Skip to content

Commit 827df4c

Browse files
Vaibhavs10Vaibhavs10
authored
Merge branch 'main' into add-link-to-local-chat
2 parents fb28cb0 + c328145 commit 827df4c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1598
-139
lines changed

‎.github/workflows/blob-publish.yml

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Blob - Version and Release
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
newversion:
7+
type: choice
8+
description: "Semantic Version Bump Type"
9+
default: patch
10+
options:
11+
- patch
12+
- minor
13+
- major
14+
15+
concurrency:
16+
group: "push-to-main"
17+
18+
defaults:
19+
run:
20+
working-directory: packages/blob
21+
22+
jobs:
23+
version_and_release:
24+
runs-on: ubuntu-latest
25+
steps:
26+
- uses: actions/checkout@v3
27+
with:
28+
# Needed to push the tag and the commit on the main branch, otherwise we get:
29+
# > Run git push --follow-tags
30+
# remote: error: GH006: Protected branch update failed for refs/heads/main.
31+
# remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
32+
token: ${{ secrets.BOT_ACCESS_TOKEN }}
33+
- run: corepack enable
34+
- uses: actions/setup-node@v3
35+
with:
36+
node-version: "20"
37+
cache: "pnpm"
38+
cache-dependency-path: |
39+
packages/blob/pnpm-lock.yaml
40+
# setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED
41+
registry-url: "https://registry.npmjs.org"
42+
- run: pnpm install
43+
- run: git config --global user.name machineuser
44+
- run: git config --global user.email [email protected]
45+
- run: |
46+
PACKAGE_VERSION=$(node -p "require('./package.json').version")
47+
BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
48+
# Update package.json with the new version
49+
node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
50+
git commit . -m "🔖 @huggingface/blob $BUMPED_VERSION"
51+
git tag "blob-v$BUMPED_VERSION"
52+
53+
- run: pnpm publish --no-git-checks .
54+
env:
55+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
56+
- run: git pull --rebase && git push --follow-tags
57+
# hack - reuse actions/setup-node@v3 just to set a new registry
58+
- uses: actions/setup-node@v3
59+
with:
60+
node-version: "20"
61+
registry-url: "https://npm.pkg.github.com"
62+
# Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
63+
# - run: pnpm publish --no-git-checks .
64+
# env:
65+
# NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
66+
- name: "Update Doc"
67+
uses: peter-evans/repository-dispatch@v2
68+
with:
69+
event-type: doc-build
70+
token: ${{ secrets.BOT_ACCESS_TOKEN }}

‎.github/workflows/dduf-publish.yml

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: DDUF - Version and Release
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
newversion:
7+
type: choice
8+
description: "Semantic Version Bump Type"
9+
default: patch
10+
options:
11+
- patch
12+
- minor
13+
- major
14+
15+
concurrency:
16+
group: "push-to-main"
17+
18+
defaults:
19+
run:
20+
working-directory: packages/dduf
21+
22+
jobs:
23+
version_and_release:
24+
runs-on: ubuntu-latest
25+
steps:
26+
- uses: actions/checkout@v3
27+
with:
28+
# Needed to push the tag and the commit on the main branch, otherwise we get:
29+
# > Run git push --follow-tags
30+
# remote: error: GH006: Protected branch update failed for refs/heads/main.
31+
# remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
32+
token: ${{ secrets.BOT_ACCESS_TOKEN }}
33+
- run: corepack enable
34+
- uses: actions/setup-node@v3
35+
with:
36+
node-version: "20"
37+
cache: "pnpm"
38+
cache-dependency-path: |
39+
packages/dduf/pnpm-lock.yaml
40+
# setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED
41+
registry-url: "https://registry.npmjs.org"
42+
- run: pnpm install
43+
- run: git config --global user.name machineuser
44+
- run: git config --global user.email [email protected]
45+
- run: |
46+
PACKAGE_VERSION=$(node -p "require('./package.json').version")
47+
BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
48+
# Update package.json with the new version
49+
node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
50+
git commit . -m "🔖 @huggingface/dduf $BUMPED_VERSION"
51+
git tag "dduf-v$BUMPED_VERSION"
52+
53+
- name: "Check Deps are published before publishing this package"
54+
run: pnpm -w check-deps blob
55+
56+
- run: pnpm publish --no-git-checks .
57+
env:
58+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
59+
- run: git pull --rebase && git push --follow-tags
60+
# hack - reuse actions/setup-node@v3 just to set a new registry
61+
- uses: actions/setup-node@v3
62+
with:
63+
node-version: "20"
64+
registry-url: "https://npm.pkg.github.com"
65+
# Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
66+
# - run: pnpm publish --no-git-checks .
67+
# env:
68+
# NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
69+
- name: "Update Doc"
70+
uses: peter-evans/repository-dispatch@v2
71+
with:
72+
event-type: doc-build
73+
token: ${{ secrets.BOT_ACCESS_TOKEN }}

‎packages/blob/.eslintignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
dist
2+
sha256.js

‎packages/blob/.prettierignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pnpm-lock.yaml
2+
# In order to avoid code samples to have tabs, they don't display well on npm
3+
README.md
4+
dist
5+
sha256.js

‎packages/blob/LICENSE

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2024 Hugging Face
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎packages/blob/README.md

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# 🤗 Hugging Face Blobs
2+
3+
Utilities to convert a string or URL to a [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob) object, whether it represents a local file or a remote URL.
4+
5+
`fetch` already returns a `Blob` object for remote URLs, but it loads the entire file in memory. This utility makes ad-hoc http range requests when calling `.slice()` on the blob.
6+
7+
## Install
8+
9+
```console
10+
pnpm add @huggingface/blob
11+
12+
npm add @huggingface/blob
13+
14+
yarn add @huggingface/blob
15+
```
16+
17+
### Deno
18+
19+
```ts
20+
// esm.sh
21+
import { FileBlob } from "https://esm.sh/@huggingface/blob/FileBlob";
22+
import { WebBlob } from "https://esm.sh/@huggingface/blob/WebBlob";
23+
import { createBlob } from "https://esm.sh/@huggingface/blob";
24+
// or npm:
25+
import { FileBlob } from "npm:@huggingface/blob/FileBlob";
26+
import { WebBlob } from "npm:@huggingface/blob/WebBlob";
27+
import { createBlob } from "npm:@huggingface/blob";
28+
```
29+
30+
## Usage
31+
32+
33+
```ts
34+
import { FileBlob } from "@huggingface/blob/FileBlob";
35+
import { WebBlob } from "@huggingface/blob/WebBlob";
36+
import { createBlob } from "@huggingface/blob";
37+
38+
const fileBlob = await FileBlob.create("path/to/file");
39+
const webBlob = await WebBlob.create("https://url/to/file");
40+
41+
const blob = await createBlob("..."); // Automatically detects if it's a file or web URL
42+
```
43+
44+
## API
45+
46+
### createBlob
47+
48+
Creates a Blob object from a string or URL. Automatically detects if it's a file or web URL.
49+
50+
```ts
51+
await createBlob("...", {
52+
/**
53+
* Custom fetch function to use, in case it resolves to a Web Blob.
54+
*
55+
* Useful for adding headers, etc.
56+
*/
57+
fetch: ...,
58+
});
59+
60+
### FileBlob
61+
62+
```ts
63+
await FileBlob.create("path/to/file");
64+
await FileBlob.create(new URL("file:///path/to/file"));
65+
```
66+
67+
### WebBlob
68+
69+
Creates a Blob object from a URL. If the file is less than 1MB (as indicated by the Content-Length header), by default it will be cached in memory in entirety upon blob creation.
70+
71+
This class is useful for large files that do not need to be loaded all at once in memory, as it makes range requests for the data.
72+
73+
```ts
74+
await WebBlob.create("https://url/to/file");
75+
await WebBlob.create(new URL("https://url/to/file"));
76+
77+
await WebBlob.create("https://url/to/file", {
78+
/**
79+
* Custom fetch function to use. Useful for adding headers, etc.
80+
*/
81+
fetch: ...,
82+
/**
83+
* If the file is less than the specified size, it will be cached in memory in entirety upon blob creation,
84+
* instead of doing range requests for the data.
85+
*
86+
* @default 1_000_000
87+
*/
88+
cacheBelow: ...
89+
})
90+
```

‎packages/blob/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export * from "./src/index";

‎packages/blob/package.json

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
{
2+
"name": "@huggingface/blob",
3+
"packageManager": "[email protected]",
4+
"version": "0.0.2",
5+
"description": "Utilities to convert URLs and files to Blobs, internally used by Hugging Face libs",
6+
"repository": "https://github.com/huggingface/huggingface.js.git",
7+
"publishConfig": {
8+
"access": "public"
9+
},
10+
"main": "./dist/index.js",
11+
"module": "./dist/index.mjs",
12+
"types": "./dist/index.d.ts",
13+
"exports": {
14+
".": {
15+
"require": "./dist/index.js",
16+
"import": "./dist/index.mjs"
17+
},
18+
"./package.json": "./package.json",
19+
"./WebBlob": {
20+
"require": "./dist/src/utils/WebBlob.js",
21+
"import": "./dist/src/utils/WebBlob.mjs"
22+
},
23+
"./FileBlob": {
24+
"require": "./dist/src/utils/FileBlob.js",
25+
"import": "./dist/src/utils/FileBlob.mjs"
26+
}
27+
},
28+
"browser": {
29+
"./src/utils/FileBlob.ts": false,
30+
"./dist/index.js": "./dist/browser/index.js",
31+
"./dist/index.mjs": "./dist/browser/index.mjs"
32+
},
33+
"source": "index.ts",
34+
"scripts": {
35+
"lint": "eslint --quiet --fix --ext .cjs,.ts .",
36+
"lint:check": "eslint --ext .cjs,.ts .",
37+
"format": "prettier --write .",
38+
"format:check": "prettier --check .",
39+
"prepublishOnly": "pnpm run build",
40+
"build": "tsup && tsc --emitDeclarationOnly --declaration && cp dist/index.d.ts dist/index.m.d.ts && cp dist/src/utils/FileBlob.d.ts dist/src/utils/FileBlob.m.d.ts && cp dist/src/utils/WebBlob.d.ts dist/src/utils/WebBlob.m.d.ts",
41+
"prepare": "pnpm run build",
42+
"test": "vitest run",
43+
"test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts",
44+
"check": "tsc"
45+
},
46+
"files": [
47+
"src",
48+
"dist",
49+
"index.ts",
50+
"tsconfig.json"
51+
],
52+
"keywords": [
53+
"huggingface",
54+
"hugging",
55+
"face",
56+
"blob",
57+
"lazy"
58+
],
59+
"author": "Hugging Face",
60+
"license": "MIT",
61+
"devDependencies": {
62+
"@types/node": "^20.11.28"
63+
}
64+
}

‎packages/blob/pnpm-lock.yaml

+22
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/blob/src/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export { createBlob } from "./utils/createBlob";

0 commit comments

Comments
 (0)