Skip to content

Commit

Permalink
Merge pull request #29 from QuantStack/reliablility
Browse files Browse the repository at this point in the history 
Improve user experience reliability
  • Loading branch information
@afshin
afshin authored Oct 23, 2024
2 parents bc44df5 + 9aab2b5 commit a3e3600
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 60 deletions.
95 changes: 62 additions & 33 deletions src/s3.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,12 @@ export const listS3Contents = async (
path?: string
): Promise<Contents.IModel> => {
const fileList: IContentsList = {};
const prefix = path ? PathExt.join(root, path) : root;

// listing contents of folder
const command = new ListObjectsV2Command({
Bucket: bucketName,
Prefix: path ? PathExt.join(root, path) : root
Prefix: prefix + (prefix ? '/' : '')
});

let isTruncated: boolean | undefined = true;
Expand Down Expand Up @@ -256,21 +257,35 @@ export const createS3Object = async (
if (options) {
body = Private.formatBody(options, fileFormat, fileType, fileMimeType);
}
let lastModified;

await s3Client.send(
new PutObjectCommand({
Bucket: bucketName,
Key: path + (PathExt.extname(name) === '' ? '/' : ''),
Body: body,
CacheControl: options ? 'no-cache' : undefined
await s3Client
.send(
new PutObjectCommand({
Bucket: bucketName,
Key: path + (PathExt.extname(name) === '' ? '/' : ''),
Body: body,
CacheControl: options ? 'no-cache' : undefined
})
)
.then(async () => {
const newFileInfo = await s3Client.send(
new HeadObjectCommand({
Bucket: bucketName,
Key: path + (PathExt.extname(name) === '' ? '/' : '')
})
);
lastModified = newFileInfo.LastModified?.toISOString();
})
);
.catch(error => {
console.error('Failed saving or creating the S3 object: ', error);
});

data = {
name: name,
path: PathExt.join(path, name),
last_modified: new Date().toISOString(),
created: new Date().toISOString(),
last_modified: lastModified ?? new Date().toISOString(),
created: lastModified ?? new Date().toISOString(),
content: path.split('.').length === 1 ? [] : body,
format: fileFormat as Contents.FileFormat,
mimetype: fileMimeType,
Expand Down Expand Up @@ -312,9 +327,9 @@ export const deleteS3Objects = async (

if (Contents) {
await Promise.all(
Contents.map(c => {
Contents.map(async c => {
// delete each file with given path
Private.deleteFile(s3Client, bucketName, c.Key!);
return Private.deleteFile(s3Client, bucketName, c.Key!);
})
);
}
Expand Down Expand Up @@ -402,28 +417,14 @@ export const renameS3Objects = async (
await s3Client.send(command);

if (Contents) {
// retrieve information of file or directory
const fileContents = await s3Client.send(
// retrieve content of file or directory
const oldFileContents = await s3Client.send(
new GetObjectCommand({
Bucket: bucketName,
Key: Contents[0].Key!
})
);

const body = await fileContents.Body?.transformToString();

data = {
name: newFileName,
path: newLocalPath,
last_modified: fileContents.LastModified!.toISOString(),
created: '',
content: body ? body : [],
format: fileFormat as Contents.FileFormat,
mimetype: fileMimeType,
size: fileContents.ContentLength!,
writable: true,
type: fileType
};
const body = await oldFileContents.Body?.transformToString();

const promises = Contents.map(async c => {
const remainingFilePath = c.Key!.substring(oldLocalPath.length);
Expand All @@ -442,13 +443,37 @@ export const renameS3Objects = async (
);
});
await Promise.all(promises);

let lastModifiedDate = new Date().toISOString();
if (!isDir) {
// retrieve last modified time for new file, does not apply to remaming directory
const newFileMetadata = await s3Client.send(
new HeadObjectCommand({
Bucket: bucketName,
Key: newLocalPath
})
);
lastModifiedDate = newFileMetadata.LastModified!.toISOString();
}

data = {
name: newFileName,
path: newLocalPath.replace(root, ''),
last_modified: lastModifiedDate,
created: '',
content: body ? body : [],
format: fileFormat as Contents.FileFormat,
mimetype: fileMimeType,
size: oldFileContents.ContentLength!,
writable: true,
type: fileType
};
}
if (isTruncated) {
isTruncated = IsTruncated;
}
command.input.ContinuationToken = NextContinuationToken;
}

return data;
};

Expand Down Expand Up @@ -479,12 +504,12 @@ export const copyS3Objects = async (
newBucketName?: string
): Promise<Contents.IModel> => {
const isDir: boolean = PathExt.extname(path) === '';
let suffix: string = '';

path = PathExt.join(root, path);
toDir = PathExt.join(root, toDir);

name = PathExt.join(toDir, name);
path = isDir ? path + '/' : path;
path = path + (isDir ? '/' : '');

// list contents of path - contents of directory or one file
const command = new ListObjectsV2Command({
Expand All @@ -500,6 +525,9 @@ export const copyS3Objects = async (

if (Contents) {
const promises = Contents.map(c => {
if (!suffix && c.Key!.search('/.emptyFolderPlaceholder') !== -1) {
suffix = '.emptyFolderPlaceholder';
}
const remainingFilePath = c.Key!.substring(path.length);
// copy each file from old directory to new location
return Private.copyFile(
Expand Down Expand Up @@ -528,7 +556,7 @@ export const copyS3Objects = async (
const newFileContents = await s3Client.send(
new GetObjectCommand({
Bucket: newBucketName ?? bucketName,
Key: name
Key: name + (suffix ? suffix : '')
})
);

Expand Down Expand Up @@ -672,6 +700,7 @@ namespace Private {
Key: PathExt.join(newPath, remainingFilePath)
})
);
console.log('copy: ', PathExt.join(newPath, remainingFilePath));
}

/**
Expand Down
47 changes: 20 additions & 27 deletions src/s3contents.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,6 @@ export class Drive implements Contents.IDrive {
oldValue: null,
newValue: data
});

return data;
}

Expand Down Expand Up @@ -428,35 +427,29 @@ export class Drive implements Contents.IDrive {
): Promise<Contents.IModel> {
let newFileName = PathExt.basename(newLocalPath);

await checkS3Object(this._s3Client, this._name, this._root, newLocalPath)
.then(async () => {
console.log('File name already exists!');
// construct new incremented name
newFileName = await this.incrementName(newLocalPath, this._name);
})
.catch(() => {
// function throws error as the file name doesn't exist
console.log("Name doesn't exist!");
})
.finally(async () => {
// once the name has been incremented if needed, proceed with the renaming
data = await renameS3Objects(
this._s3Client,
this._name,
this._root,
oldLocalPath,
newLocalPath,
newFileName,
this._registeredFileTypes
);
});
try {
await checkS3Object(this._s3Client, this._name, this._root, newLocalPath);
newFileName = await this.incrementName(newLocalPath, this._name);
} catch (error) {
// HEAD request failed for this file name, continue, as name doesn't already exist.
} finally {
data = await renameS3Objects(
this._s3Client,
this._name,
this._root,
oldLocalPath,
newLocalPath,
newFileName,
this._registeredFileTypes
);
}

Contents.validateContentsModel(data);
this._fileChanged.emit({
type: 'rename',
oldValue: { path: oldLocalPath },
newValue: data
});
Contents.validateContentsModel(data);
return data;
}

Expand Down Expand Up @@ -534,12 +527,12 @@ export class Drive implements Contents.IDrive {
options
);

Contents.validateContentsModel(data);
this._fileChanged.emit({
type: 'save',
oldValue: null,
newValue: data
});
Contents.validateContentsModel(data);
return data;
}

Expand Down Expand Up @@ -605,12 +598,12 @@ export class Drive implements Contents.IDrive {
this._registeredFileTypes
);

Contents.validateContentsModel(data);
this._fileChanged.emit({
type: 'new',
oldValue: null,
newValue: data
});
Contents.validateContentsModel(data);
return data;
}

Expand Down Expand Up @@ -773,7 +766,7 @@ export class Drive implements Contents.IDrive {
root = PathExt.removeSlash(PathExt.normalize(root));
// check if directory exists within bucket
try {
checkS3Object(this._s3Client, this._name, root);
await checkS3Object(this._s3Client, this._name, root);
// the directory exists, root is formatted correctly
return root;
} catch (error) {
Expand Down

0 comments on commit a3e3600

Please sign in to comment.