diff --git a/functional-samples/ai.gemini-in-the-cloud/images/icon128.png b/functional-samples/ai.gemini-in-the-cloud/images/icon128.png index 7879061999..6250bc6823 100644 Binary files a/functional-samples/ai.gemini-in-the-cloud/images/icon128.png and b/functional-samples/ai.gemini-in-the-cloud/images/icon128.png differ diff --git a/functional-samples/ai.gemini-in-the-cloud/images/icon16.png b/functional-samples/ai.gemini-in-the-cloud/images/icon16.png index a8eb458c5e..8a9871e31a 100644 Binary files a/functional-samples/ai.gemini-in-the-cloud/images/icon16.png and b/functional-samples/ai.gemini-in-the-cloud/images/icon16.png differ diff --git a/functional-samples/ai.gemini-in-the-cloud/images/icon32.png b/functional-samples/ai.gemini-in-the-cloud/images/icon32.png index 838c426c8e..f8ba754a31 100644 Binary files a/functional-samples/ai.gemini-in-the-cloud/images/icon32.png and b/functional-samples/ai.gemini-in-the-cloud/images/icon32.png differ diff --git a/functional-samples/ai.gemini-in-the-cloud/images/icon48.png b/functional-samples/ai.gemini-in-the-cloud/images/icon48.png index 932bdfe4b3..3dc878b6c1 100644 Binary files a/functional-samples/ai.gemini-in-the-cloud/images/icon48.png and b/functional-samples/ai.gemini-in-the-cloud/images/icon48.png differ diff --git a/functional-samples/ai.gemini-in-the-cloud/images/record-audio-24.svg b/functional-samples/ai.gemini-in-the-cloud/images/record-audio-24.svg new file mode 100644 index 0000000000..4ccf9865a3 --- /dev/null +++ b/functional-samples/ai.gemini-in-the-cloud/images/record-audio-24.svg @@ -0,0 +1 @@ + diff --git a/functional-samples/ai.gemini-in-the-cloud/images/screenshot-monitor-24.svg b/functional-samples/ai.gemini-in-the-cloud/images/screenshot-monitor-24.svg new file mode 100644 index 0000000000..06a2aad45e --- /dev/null +++ b/functional-samples/ai.gemini-in-the-cloud/images/screenshot-monitor-24.svg @@ -0,0 +1 @@ + diff --git a/functional-samples/ai.gemini-in-the-cloud/manifest.json b/functional-samples/ai.gemini-in-the-cloud/manifest.json index 9f7574c3de..90d0730a40 100644 --- a/functional-samples/ai.gemini-in-the-cloud/manifest.json +++ b/functional-samples/ai.gemini-in-the-cloud/manifest.json @@ -6,7 +6,8 @@ "background": { "service_worker": "background.js" }, - "permissions": ["sidePanel"], + "permissions": ["activeTab", "scripting", "sidePanel"], + "host_permissions": [""], "side_panel": { "default_path": "sidepanel/index.html" }, diff --git a/functional-samples/ai.gemini-in-the-cloud/package-lock.json b/functional-samples/ai.gemini-in-the-cloud/package-lock.json index 8aafc45618..e4caa659af 100644 --- a/functional-samples/ai.gemini-in-the-cloud/package-lock.json +++ b/functional-samples/ai.gemini-in-the-cloud/package-lock.json @@ -8,23 +8,23 @@ "name": "Chrome Extensions Gemini Demo", "version": "1.0", "devDependencies": { - "@google/generative-ai": "0.15.0", - "rollup": "4.19.0" + "@google/generative-ai": "0.17.1", + "rollup": "4.21.2" } }, "node_modules/@google/generative-ai": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.15.0.tgz", - "integrity": "sha512-zs37judcTYFJf1U7tnuqnh7gdzF6dcWj9pNRxjA5JTONRoiQ0htrRdbefRFiewOIfXwhun5t9hbd2ray7812eQ==", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.17.1.tgz", + "integrity": "sha512-TgWz02c5l2XJlEDys81UVat5+Qg9xqmYah7tQt6xlsBwFvzIFPz64aZFGd1av2sxT22NsssqLATjNsusAIJICA==", "dev": true, "engines": { "node": ">=18.0.0" } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.19.0.tgz", - "integrity": "sha512-JlPfZ/C7yn5S5p0yKk7uhHTTnFlvTgLetl2VxqE518QgyM7C9bSfFTYvB/Q/ftkq0RIPY4ySxTz+/wKJ/dXC0w==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.21.2.tgz", + "integrity": "sha512-fSuPrt0ZO8uXeS+xP3b+yYTCBUd05MoSp2N/MFOgjhhUhMmchXlpTQrTpI8T+YAwAQuK7MafsCOxW7VrPMrJcg==", "cpu": [ "arm" ], @@ -35,9 +35,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.19.0.tgz", - "integrity": "sha512-RDxUSY8D1tWYfn00DDi5myxKgOk6RvWPxhmWexcICt/MEC6yEMr4HNCu1sXXYLw8iAsg0D44NuU+qNq7zVWCrw==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.21.2.tgz", + "integrity": "sha512-xGU5ZQmPlsjQS6tzTTGwMsnKUtu0WVbl0hYpTPauvbRAnmIvpInhJtgjj3mcuJpEiuUw4v1s4BimkdfDWlh7gA==", "cpu": [ "arm64" ], @@ -48,9 +48,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.19.0.tgz", - "integrity": "sha512-emvKHL4B15x6nlNTBMtIaC9tLPRpeA5jMvRLXVbl/W9Ie7HhkrE7KQjvgS9uxgatL1HmHWDXk5TTS4IaNJxbAA==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.21.2.tgz", + "integrity": "sha512-99AhQ3/ZMxU7jw34Sq8brzXqWH/bMnf7ZVhvLk9QU2cOepbQSVTns6qoErJmSiAvU3InRqC2RRZ5ovh1KN0d0Q==", "cpu": [ "arm64" ], @@ -61,9 +61,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.19.0.tgz", - "integrity": "sha512-fO28cWA1dC57qCd+D0rfLC4VPbh6EOJXrreBmFLWPGI9dpMlER2YwSPZzSGfq11XgcEpPukPTfEVFtw2q2nYJg==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.21.2.tgz", + "integrity": "sha512-ZbRaUvw2iN/y37x6dY50D8m2BnDbBjlnMPotDi/qITMJ4sIxNY33HArjikDyakhSv0+ybdUxhWxE6kTI4oX26w==", "cpu": [ "x64" ], @@ -74,9 +74,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.19.0.tgz", - "integrity": "sha512-2Rn36Ubxdv32NUcfm0wB1tgKqkQuft00PtM23VqLuCUR4N5jcNWDoV5iBC9jeGdgS38WK66ElncprqgMUOyomw==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.21.2.tgz", + "integrity": "sha512-ztRJJMiE8nnU1YFcdbd9BcH6bGWG1z+jP+IPW2oDUAPxPjo9dverIOyXz76m6IPA6udEL12reYeLojzW2cYL7w==", "cpu": [ "arm" ], @@ -87,9 +87,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.19.0.tgz", - "integrity": "sha512-gJuzIVdq/X1ZA2bHeCGCISe0VWqCoNT8BvkQ+BfsixXwTOndhtLUpOg0A1Fcx/+eA6ei6rMBzlOz4JzmiDw7JQ==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.21.2.tgz", + "integrity": "sha512-flOcGHDZajGKYpLV0JNc0VFH361M7rnV1ee+NTeC/BQQ1/0pllYcFmxpagltANYt8FYf9+kL6RSk80Ziwyhr7w==", "cpu": [ "arm" ], @@ -100,9 +100,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.19.0.tgz", - "integrity": "sha512-0EkX2HYPkSADo9cfeGFoQ7R0/wTKb7q6DdwI4Yn/ULFE1wuRRCHybxpl2goQrx4c/yzK3I8OlgtBu4xvted0ug==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.21.2.tgz", + "integrity": "sha512-69CF19Kp3TdMopyteO/LJbWufOzqqXzkrv4L2sP8kfMaAQ6iwky7NoXTp7bD6/irKgknDKM0P9E/1l5XxVQAhw==", "cpu": [ "arm64" ], @@ -113,9 +113,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.19.0.tgz", - "integrity": "sha512-GlIQRj9px52ISomIOEUq/IojLZqzkvRpdP3cLgIE1wUWaiU5Takwlzpz002q0Nxxr1y2ZgxC2obWxjr13lvxNQ==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.21.2.tgz", + "integrity": "sha512-48pD/fJkTiHAZTnZwR0VzHrao70/4MlzJrq0ZsILjLW/Ab/1XlVUStYyGt7tdyIiVSlGZbnliqmult/QGA2O2w==", "cpu": [ "arm64" ], @@ -126,9 +126,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.19.0.tgz", - "integrity": "sha512-N6cFJzssruDLUOKfEKeovCKiHcdwVYOT1Hs6dovDQ61+Y9n3Ek4zXvtghPPelt6U0AH4aDGnDLb83uiJMkWYzQ==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.21.2.tgz", + "integrity": "sha512-cZdyuInj0ofc7mAQpKcPR2a2iu4YM4FQfuUzCVA2u4HI95lCwzjoPtdWjdpDKyHxI0UO82bLDoOaLfpZ/wviyQ==", "cpu": [ "ppc64" ], @@ -139,9 +139,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.19.0.tgz", - "integrity": "sha512-2DnD3mkS2uuam/alF+I7M84koGwvn3ZVD7uG+LEWpyzo/bq8+kKnus2EVCkcvh6PlNB8QPNFOz6fWd5N8o1CYg==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.21.2.tgz", + "integrity": "sha512-RL56JMT6NwQ0lXIQmMIWr1SW28z4E4pOhRRNqwWZeXpRlykRIlEpSWdsgNWJbYBEWD84eocjSGDu/XxbYeCmwg==", "cpu": [ "riscv64" ], @@ -152,9 +152,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.19.0.tgz", - "integrity": "sha512-D6pkaF7OpE7lzlTOFCB2m3Ngzu2ykw40Nka9WmKGUOTS3xcIieHe82slQlNq69sVB04ch73thKYIWz/Ian8DUA==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.21.2.tgz", + "integrity": "sha512-PMxkrWS9z38bCr3rWvDFVGD6sFeZJw4iQlhrup7ReGmfn7Oukrr/zweLhYX6v2/8J6Cep9IEA/SmjXjCmSbrMQ==", "cpu": [ "s390x" ], @@ -165,9 +165,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.19.0.tgz", - "integrity": "sha512-HBndjQLP8OsdJNSxpNIN0einbDmRFg9+UQeZV1eiYupIRuZsDEoeGU43NQsS34Pp166DtwQOnpcbV/zQxM+rWA==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.21.2.tgz", + "integrity": "sha512-B90tYAUoLhU22olrafY3JQCFLnT3NglazdwkHyxNDYF/zAxJt5fJUB/yBoWFoIQ7SQj+KLe3iL4BhOMa9fzgpw==", "cpu": [ "x64" ], @@ -178,9 +178,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.19.0.tgz", - "integrity": "sha512-HxfbvfCKJe/RMYJJn0a12eiOI9OOtAUF4G6ozrFUK95BNyoJaSiBjIOHjZskTUffUrB84IPKkFG9H9nEvJGW6A==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.21.2.tgz", + "integrity": "sha512-7twFizNXudESmC9oneLGIUmoHiiLppz/Xs5uJQ4ShvE6234K0VB1/aJYU3f/4g7PhssLGKBVCC37uRkkOi8wjg==", "cpu": [ "x64" ], @@ -191,9 +191,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.19.0.tgz", - "integrity": "sha512-HxDMKIhmcguGTiP5TsLNolwBUK3nGGUEoV/BO9ldUBoMLBssvh4J0X8pf11i1fTV7WShWItB1bKAKjX4RQeYmg==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.21.2.tgz", + "integrity": "sha512-9rRero0E7qTeYf6+rFh3AErTNU1VCQg2mn7CQcI44vNUWM9Ze7MSRS/9RFuSsox+vstRt97+x3sOhEey024FRQ==", "cpu": [ "arm64" ], @@ -204,9 +204,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.19.0.tgz", - "integrity": "sha512-xItlIAZZaiG/u0wooGzRsx11rokP4qyc/79LkAOdznGRAbOFc+SfEdfUOszG1odsHNgwippUJavag/+W/Etc6Q==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.21.2.tgz", + "integrity": "sha512-5rA4vjlqgrpbFVVHX3qkrCo/fZTj1q0Xxpg+Z7yIo3J2AilW7t2+n6Q8Jrx+4MrYpAnjttTYF8rr7bP46BPzRw==", "cpu": [ "ia32" ], @@ -217,9 +217,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.19.0.tgz", - "integrity": "sha512-xNo5fV5ycvCCKqiZcpB65VMR11NJB+StnxHz20jdqRAktfdfzhgjTiJ2doTDQE/7dqGaV5I7ZGqKpgph6lCIag==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.21.2.tgz", + "integrity": "sha512-6UUxd0+SKomjdzuAcp+HAmxw1FlGBnl1v2yEPSabtx4lBfdXHDVsW7+lQkgz9cNFJGY3AWR7+V8P5BqkD9L9nA==", "cpu": [ "x64" ], @@ -250,9 +250,9 @@ } }, "node_modules/rollup": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.19.0.tgz", - "integrity": "sha512-5r7EYSQIowHsK4eTZ0Y81qpZuJz+MUuYeqmmYmRMl1nwhdmbiYqt5jwzf6u7wyOzJgYqtCRMtVRKOtHANBz7rA==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.21.2.tgz", + "integrity": "sha512-e3TapAgYf9xjdLvKQCkQTnbTKd4a6jwlpQSJJFokHGaX2IVjoEqkIIhiQfqsi0cdwlOD+tQGuOd5AJkc5RngBw==", "dev": true, "dependencies": { "@types/estree": "1.0.5" @@ -265,22 +265,22 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.19.0", - "@rollup/rollup-android-arm64": "4.19.0", - "@rollup/rollup-darwin-arm64": "4.19.0", - "@rollup/rollup-darwin-x64": "4.19.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.19.0", - "@rollup/rollup-linux-arm-musleabihf": "4.19.0", - "@rollup/rollup-linux-arm64-gnu": "4.19.0", - "@rollup/rollup-linux-arm64-musl": "4.19.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.19.0", - "@rollup/rollup-linux-riscv64-gnu": "4.19.0", - "@rollup/rollup-linux-s390x-gnu": "4.19.0", - "@rollup/rollup-linux-x64-gnu": "4.19.0", - "@rollup/rollup-linux-x64-musl": "4.19.0", - "@rollup/rollup-win32-arm64-msvc": "4.19.0", - "@rollup/rollup-win32-ia32-msvc": "4.19.0", - "@rollup/rollup-win32-x64-msvc": "4.19.0", + "@rollup/rollup-android-arm-eabi": "4.21.2", + "@rollup/rollup-android-arm64": "4.21.2", + "@rollup/rollup-darwin-arm64": "4.21.2", + "@rollup/rollup-darwin-x64": "4.21.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.21.2", + "@rollup/rollup-linux-arm-musleabihf": "4.21.2", + "@rollup/rollup-linux-arm64-gnu": "4.21.2", + "@rollup/rollup-linux-arm64-musl": "4.21.2", + "@rollup/rollup-linux-powerpc64le-gnu": "4.21.2", + "@rollup/rollup-linux-riscv64-gnu": "4.21.2", + "@rollup/rollup-linux-s390x-gnu": "4.21.2", + "@rollup/rollup-linux-x64-gnu": "4.21.2", + "@rollup/rollup-linux-x64-musl": "4.21.2", + "@rollup/rollup-win32-arm64-msvc": "4.21.2", + "@rollup/rollup-win32-ia32-msvc": "4.21.2", + "@rollup/rollup-win32-x64-msvc": "4.21.2", "fsevents": "~2.3.2" } } diff --git a/functional-samples/ai.gemini-in-the-cloud/package.json b/functional-samples/ai.gemini-in-the-cloud/package.json index 8b856c619b..fa905a7979 100644 --- a/functional-samples/ai.gemini-in-the-cloud/package.json +++ b/functional-samples/ai.gemini-in-the-cloud/package.json @@ -6,7 +6,7 @@ }, "private": true, "devDependencies": { - "@google/generative-ai": "0.15.0", - "rollup": "4.19.0" + "@google/generative-ai": "0.17.1", + "rollup": "4.21.2" } } diff --git a/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.css b/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.css index d88a10d5f0..02b230ad2d 100644 --- a/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.css +++ b/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.css @@ -31,6 +31,10 @@ button.primary { color: white; } +button > img { + vertical-align: middle; +} + button.secondary { background: #ccc; color: black; diff --git a/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.html b/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.html index 9c6deff5ca..9c69f49e71 100644 --- a/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.html +++ b/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.html @@ -4,13 +4,13 @@ -

Google Gemini

+
Google Gemini >Temperature: 1
- +
+ + + + + + + +
+
+ + +
diff --git a/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.js b/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.js index 73f7180d8e..c9c2785dd7 100644 --- a/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.js +++ b/functional-samples/ai.gemini-in-the-cloud/sidepanel/index.js @@ -4,6 +4,9 @@ import { HarmCategory } from '../node_modules/@google/generative-ai/dist/index.mjs'; +import TabAudioRecorder from './tab-audio-recorder'; +import getScreenshot from './screenshot'; + // Important! Do not expose your API in your extension code. You have to // options: // @@ -14,21 +17,143 @@ import { // // It is only OK to put your API key into this file if you're the only // user of your extension or for testing. -const apiKey = '...'; +const apiKey = 'AIzaSyAiTmPF3fbapEgNtpkVyAOxNb0GZbvLfyE'; let genAI = null; let model = null; let generationConfig = { temperature: 1 }; +let promptFiles = []; +let screenshotCount = 0; +let audioRecordingCount = 0; + +const tabAudioRecorder = new TabAudioRecorder(); const inputPrompt = document.body.querySelector('#input-prompt'); const buttonPrompt = document.body.querySelector('#button-prompt'); +const buttonReset = document.body.querySelector('#button-reset'); const elementResponse = document.body.querySelector('#response'); const elementLoading = document.body.querySelector('#loading'); const elementError = document.body.querySelector('#error'); const sliderTemperature = document.body.querySelector('#temperature'); const labelTemperature = document.body.querySelector('#label-temperature'); +const elementAudioFile = document.getElementById('audio-file'); +const buttonAudioFile = document.getElementById('button-record-audio'); +const elementImageFile = document.getElementById('image-file'); +const buttonImageFile = document.getElementById('button-add-image'); +const listFiles = document.getElementById('files-list'); +const buttonCaptureAudio = document.getElementById('button-capture-audio'); +const buttonCaptureAudioStop = document.getElementById( + 'button-capture-audio-stop' +); +const buttonCaptureImage = document.getElementById('button-capture-image'); + +buttonImageFile.addEventListener( + 'click', + () => { + elementImageFile.click(); + }, + false +); +elementImageFile.addEventListener('change', async () => { + uploadFiles(elementImageFile.files, 'image'); +}); + +buttonAudioFile.addEventListener( + 'click', + () => { + elementAudioFile.click(); + }, + false +); + +elementAudioFile.addEventListener('change', async () => { + uploadFiles(elementAudioFile.files, 'audio'); +}); + +buttonCaptureImage.addEventListener('click', async () => { + try { + const screenshot = await getScreenshot(); + console.log('screenshot', screenshot); + const fileName = + screenshotCount > 0 ? `Screenshot_${screenshotCount + 1}` : 'Screenshot'; + renderUploadedFile(fileName, 'image', screenshot.data); + const imagePart = { + inlineData: { data: screenshot.base64, mimeType: 'image/png' } + }; + promptFiles.push({ + name: fileName, + preview: screenshot, + mimeType: 'image/png', + imagePart + }); + screenshotCount++; + } catch (e) { + console.log(e); + alert(e.message); + } +}); + +buttonCaptureAudio.addEventListener('click', async () => { + try { + await tabAudioRecorder.start(); + hide(buttonCaptureAudio); + show(buttonCaptureAudioStop); + } catch (e) { + console.error(e); + alert(e.message); + } +}); + +buttonCaptureAudioStop.addEventListener('click', async () => { + tabAudioRecorder.stop(); + show(buttonCaptureAudio); + hide(buttonCaptureAudioStop); +}); + +async function uploadFiles(files, type) { + for (const file of files) { + // getting base64 from file to render in DOM + const base64 = await getBase64(file); + // generating content model for Gemini Google AI + const imagePart = await fileToGenerativePart(file); + renderUploadedFile(file.name, type, base64); + promptFiles.push({ + name: file.name, + preview: base64, + imagePart + }); + } +} + +function renderUploadedFile(name, type) { + const fileElement = document.createElement('LI'); + fileElement.textContent = name; + fileElement.classList.add(type); + listFiles.appendChild(fileElement); +} + +function getBase64(file) { + return new Promise(function (resolve, reject) { + let reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => resolve(reader.result); + reader.onerror = (error) => reject('Error: ', error); + }); +} + +async function fileToGenerativePart(file) { + const base64EncodedDataPromise = new Promise((resolve) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result.split(',')[1]); + reader.readAsDataURL(file); + }); + + return { + inlineData: { data: await base64EncodedDataPromise, mimeType: file.type } + }; +} function initModel(generationConfig) { const safetySettings = [ @@ -48,7 +173,10 @@ function initModel(generationConfig) { async function runPrompt(prompt) { try { - const result = await model.generateContent(prompt); + const result = await model.generateContent([ + prompt, + ...promptFiles.map((f) => f.imagePart) + ]); const response = await result.response; return response.text(); } catch (e) { @@ -87,6 +215,16 @@ buttonPrompt.addEventListener('click', async () => { } }); +buttonReset.addEventListener('click', async () => { + promptFiles = []; + listFiles.textContent = ''; + model = null; + elementAudioFile.value = null; + elementImageFile.value = null; + screenshotCount = 0; + audioRecordingCount = 0; +}); + function showLoading() { hide(elementResponse); hide(elementError); diff --git a/functional-samples/ai.gemini-in-the-cloud/sidepanel/screenshot.js b/functional-samples/ai.gemini-in-the-cloud/sidepanel/screenshot.js new file mode 100644 index 0000000000..c1358b11e9 --- /dev/null +++ b/functional-samples/ai.gemini-in-the-cloud/sidepanel/screenshot.js @@ -0,0 +1,24 @@ +const getScreenshot = async () => { + const imageDataUri = await chrome.tabs.captureVisibleTab({ format: 'png' }); + const byteString = atob(imageDataUri.split(',')[1]); + const ab = new ArrayBuffer(byteString.length); + const ia = new Uint8Array(ab); + for (let i = 0; i < byteString.length; i++) { + ia[i] = byteString.charCodeAt(i); + } + return { + data: ia, + base64: await blobToBase64(new Blob([ia], { type: 'image/png' })) + }; +}; + +const blobToBase64 = async (imageDataUri) => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.readAsDataURL(imageDataUri); + reader.onloadend = () => resolve(reader.result.split(',')[1]); + reader.onerror = reject; + }); +}; + +export default getScreenshot; diff --git a/functional-samples/ai.gemini-in-the-cloud/sidepanel/tab-audio-recorder.js b/functional-samples/ai.gemini-in-the-cloud/sidepanel/tab-audio-recorder.js new file mode 100644 index 0000000000..91600eee7f --- /dev/null +++ b/functional-samples/ai.gemini-in-the-cloud/sidepanel/tab-audio-recorder.js @@ -0,0 +1,60 @@ +export default class TabAudioRecorder { + constructor() { + this.recorder = null; + this.data = []; + } + + async start() { + /* + const currentTab = await chrome.tabs.query({ + active: true, + currentWindow: true + }); + console.log('currentab', currentTab); + + const streamId = await chrome.tabCapture.getMediaStreamId({ + targetTabId: currentTab.id + }); + + const media = await navigator.mediaDevices.getUserMedia({ + audio: { + mandatory: { + chromeMediaSource: 'tab', + chromeMediaSourceId: streamId + } + } + }); + */ + + const media = await navigator.mediaDevices.getDisplayMedia({ + audio: true, + video: false + }); + + // Continue to play the captured audio to the user. + const output = new AudioContext(); + const source = output.createMediaStreamSource(media); + source.connect(output.destination); + // Start recording. + this.recorder = new MediaRecorder(media, { mimeType: 'audio/wav' }); + this.recorder.ondataavailable = (event) => this.data.push(event.data); + this.recorder.onstop = () => { + const blob = new Blob(this.data, { type: 'video/wav' }); + window.open(URL.createObjectURL(blob), '_blank'); + + // Clear state ready for next recording + this.recorder = undefined; + this.data = []; + }; + this.recorder.start(); + } + + stop() { + if (!this.recorder) { + return; + } + this.recorder.stop(); + // Stopping the tracks makes sure the recording icon in the tab is removed. + this.recorder.stream.getTracks().forEach((t) => t.stop()); + } +}