← back to Exo
add support for image upload to tinychat for vision models
af1c7ce327cf0f25fa597d68751b0157d1136287 · 2024-07-30 20:01:35 +0100 · Alex Cheema
Files touched
M tinychat/examples/tinychat/index.cssM tinychat/examples/tinychat/index.htmlM tinychat/examples/tinychat/index.js
Diff
commit af1c7ce327cf0f25fa597d68751b0157d1136287
Author: Alex Cheema <alexcheema123@gmail.com>
Date: Tue Jul 30 20:01:35 2024 +0100
add support for image upload to tinychat for vision models
---
tinychat/examples/tinychat/index.css | 70 ++++++++++++++++++++++++++++++++++
tinychat/examples/tinychat/index.html | 13 ++++++-
tinychat/examples/tinychat/index.js | 71 +++++++++++++++++++++++++++++++++--
3 files changed, 150 insertions(+), 4 deletions(-)
diff --git a/tinychat/examples/tinychat/index.css b/tinychat/examples/tinychat/index.css
index 84b04b86..b9754683 100644
--- a/tinychat/examples/tinychat/index.css
+++ b/tinychat/examples/tinychat/index.css
@@ -309,4 +309,74 @@ p {
outline: none;
border-color: #007bff;
box-shadow: 0 0 0 2px rgba(0,123,255,.25);
+}
+
+/* Image upload button styles */
+.image-input-button {
+ background-color: var(--secondary-color);
+ color: var(--foreground-color);
+ border: none;
+ border-radius: 50%;
+ width: 40px;
+ height: 40px;
+ font-size: 18px;
+ cursor: pointer;
+ transition: all 0.3s ease;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ margin-right: 10px;
+}
+
+.image-input-button:hover {
+ background-color: var(--secondary-color-transparent);
+ transform: scale(1.1);
+}
+
+.image-input-button:focus {
+ outline: none;
+ box-shadow: 0 0 0 3px rgba(var(--secondary-color-rgb), 0.5);
+}
+
+.image-input-button i {
+ transition: all 0.3s ease;
+}
+
+.image-input-button:hover i {
+ transform: scale(1.2);
+}
+
+/* Hidden file input styles */
+#image-upload {
+ display: none;
+}
+
+.image-preview-container {
+ position: relative;
+ display: inline-block;
+ margin-right: 10px;
+}
+
+.image-preview {
+ max-width: 100px;
+ max-height: 100px;
+ object-fit: cover;
+ border-radius: 5px;
+}
+
+.remove-image-button {
+ position: absolute;
+ top: -5px;
+ right: -5px;
+ background-color: rgba(255, 255, 255, 0.8);
+ border: none;
+ border-radius: 50%;
+ padding: 2px 5px;
+ cursor: pointer;
+}
+
+.message > p > img {
+ max-width: 100%;
+ max-height: 100%;
+ object-fit: contain;
}
\ No newline at end of file
diff --git a/tinychat/examples/tinychat/index.html b/tinychat/examples/tinychat/index.html
index e45c7f47..ea79704a 100644
--- a/tinychat/examples/tinychat/index.html
+++ b/tinychat/examples/tinychat/index.html
@@ -44,6 +44,7 @@
<option value="mistral-nemo">Mistral Nemo</option>
<option value="mistral-large">Mistral Large</option>
<option value="deepseek-coder-v2-lite">Deepseek Coder V2 Lite</option>
+ <option value="llava-1.5-7b-hf">LLaVa 1.5 7B (Vision Model)</option>
</select>
</div>
<div class="home centered" x-show="home === 0" x-transition x-effect="
@@ -159,6 +160,16 @@
</span>
</div>
<div class="input">
+ <button x-show="cstate.selectedModel === 'llava-1.5-7b-hf'" class="image-input-button" @click="$refs.imageUpload.click()">
+ <i class="fas fa-image"></i>
+ </button>
+ <input x-ref="imageUpload" type="file" id="image-upload" accept="image/*" @change="$data.handleImageUpload($event)" style="display: none;">
+ <div x-show="imagePreview" class="image-preview-container">
+ <img :src="imagePreview" alt="Uploaded Image" class="image-preview">
+ <button @click="imagePreview = null; imageUrl = null;" class="remove-image-button">
+ <i class="fas fa-times"></i>
+ </button>
+ </div>
<textarea x-ref="inputForm" id="input-form" class="input-form" autofocus rows=1 x-autosize
:placeholder="generating ? 'Generating...' : 'Say something'" :disabled="generating" @input="
home = (home === 0) ? 1 : home
@@ -187,4 +198,4 @@
</main>
</body>
-</html>
+</html>
\ No newline at end of file
diff --git a/tinychat/examples/tinychat/index.js b/tinychat/examples/tinychat/index.js
index fa69007d..1a15be04 100644
--- a/tinychat/examples/tinychat/index.js
+++ b/tinychat/examples/tinychat/index.js
@@ -4,6 +4,7 @@ document.addEventListener("alpine:init", () => {
cstate: {
time: null,
messages: [],
+ selectedModel: 'llama-3.1-8b',
},
// historical state
@@ -18,6 +19,9 @@ document.addEventListener("alpine:init", () => {
tokens_per_second: 0,
total_tokens: 0,
+ // image handling
+ imagePreview: null,
+
removeHistory(cstate) {
const index = this.histories.findIndex((state) => {
return state.time === cstate.time;
@@ -28,10 +32,28 @@ document.addEventListener("alpine:init", () => {
}
},
+ async handleImageUpload(event) {
+ const file = event.target.files[0];
+ if (file) {
+ const reader = new FileReader();
+ reader.onload = (e) => {
+ this.imagePreview = e.target.result;
+ this.imageUrl = e.target.result; // Store the image URL
+ // Add image preview to the chat
+ this.cstate.messages.push({
+ role: "user",
+ content: ``,
+ });
+ };
+ reader.readAsDataURL(file);
+ }
+ },
+
+
async handleSend() {
const el = document.getElementById("input-form");
const value = el.value.trim();
- if (!value) return;
+ if (!value && !this.imagePreview) return;
if (this.generating) return;
this.generating = true;
@@ -41,7 +63,9 @@ document.addEventListener("alpine:init", () => {
window.history.pushState({}, "", "/");
// add message to list
- this.cstate.messages.push({ role: "user", content: value });
+ if (value) {
+ this.cstate.messages.push({ role: "user", content: value });
+ }
// clear textarea
el.value = "";
@@ -54,10 +78,51 @@ document.addEventListener("alpine:init", () => {
let tokens = 0;
this.tokens_per_second = 0;
+ // prepare messages for API request
+ const apiMessages = this.cstate.messages.map(msg => {
+ if (msg.content.startsWith('![Uploaded Image]')) {
+ return {
+ role: "user",
+ content: [
+ {
+ type: "image_url",
+ image_url: {
+ url: this.imageUrl
+ }
+ }
+ ]
+ };
+ } else {
+ return {
+ role: msg.role,
+ content: [
+ {
+ type: "text",
+ text: msg.content
+ }
+ ]
+ };
+ }
+ });
+
+ // If there's an image URL, add it to all messages
+ if (this.imageUrl) {
+ apiMessages.forEach(msg => {
+ if (!msg.content.some(content => content.type === "image_url")) {
+ msg.content.push({
+ type: "image_url",
+ image_url: {
+ url: this.imageUrl
+ }
+ });
+ }
+ });
+ }
+
// start receiving server sent events
let gottenFirstChunk = false;
for await (
- const chunk of this.openaiChatCompletion(this.cstate.selectedModel, this.cstate.messages)
+ const chunk of this.openaiChatCompletion(this.cstate.selectedModel, apiMessages)
) {
if (!gottenFirstChunk) {
this.cstate.messages.push({ role: "assistant", content: "" });
← 0d45a855 increase max request size to send raw images, make image dow
·
back to Exo
·
use AutoProcessor with use_fast=False since there's a bug wi 2d200009 →