From ccc783ab6a00841a7041c454e77afb472d76999e Mon Sep 17 00:00:00 2001 From: Lucas Reis Date: Sun, 4 May 2025 20:37:04 +0200 Subject: [PATCH] feat(s3): stream s3 content over a zip file (#822) * chore: update package.json and enhance file handling in backend * refactor: remove StreamResponseFilter and update getZip method for improved file handling - Deleted StreamResponseFilter as it was no longer needed. - Updated getZip method in LocalFileService to return a Promise and handle errors more effectively. - Adjusted getZip method in FileService to await the storage service's getZip call. * refactor: remove unused UseFilters import in file.controller.ts --- backend/src/file/file.controller.ts | 2 +- backend/src/file/file.service.ts | 4 +- backend/src/file/local.service.ts | 17 +++- backend/src/file/s3.service.ts | 97 +++++++++++++++++++- frontend/src/i18n/translations/en-US.ts | 37 +++++--- frontend/src/pages/share/[shareId]/index.tsx | 20 ++++ package.json | 4 + 7 files changed, 159 insertions(+), 22 deletions(-) diff --git a/backend/src/file/file.controller.ts b/backend/src/file/file.controller.ts index feb036d..e7d4e96 100644 --- a/backend/src/file/file.controller.ts +++ b/backend/src/file/file.controller.ts @@ -54,7 +54,7 @@ export class FileController { @Res({ passthrough: true }) res: Response, @Param("shareId") shareId: string, ) { - const zipStream = this.fileService.getZip(shareId); + const zipStream = await this.fileService.getZip(shareId); res.set({ "Content-Type": "application/zip", diff --git a/backend/src/file/file.service.ts b/backend/src/file/file.service.ts index e5de341..fc20468 100644 --- a/backend/src/file/file.service.ts +++ b/backend/src/file/file.service.ts @@ -59,9 +59,9 @@ export class FileService { return storageService.deleteAllFiles(shareId); } - getZip(shareId: string) { + async getZip(shareId: string): Promise { const storageService = this.getStorageService(); - return storageService.getZip(shareId) as Readable; + return await storageService.getZip(shareId); } private async streamToUint8Array(stream: Readable): Promise { diff --git a/backend/src/file/local.service.ts b/backend/src/file/local.service.ts index 92c68e2..e545a5a 100644 --- a/backend/src/file/local.service.ts +++ b/backend/src/file/local.service.ts @@ -14,6 +14,7 @@ import { ConfigService } from "src/config/config.service"; import { PrismaService } from "src/prisma/prisma.service"; import { validate as isValidUUID } from "uuid"; import { SHARE_DIRECTORY } from "../constants"; +import { Readable } from "stream"; @Injectable() export class LocalFileService { @@ -155,7 +156,19 @@ export class LocalFileService { }); } - getZip(shareId: string) { - return createReadStream(`${SHARE_DIRECTORY}/${shareId}/archive.zip`); + async getZip(shareId: string): Promise { + return new Promise((resolve, reject) => { + const zipStream = createReadStream( + `${SHARE_DIRECTORY}/${shareId}/archive.zip`, + ); + + zipStream.on("error", (err) => { + reject(new InternalServerErrorException(err)); + }); + + zipStream.on("open", () => { + resolve(zipStream); + }); + }); } } diff --git a/backend/src/file/s3.service.ts b/backend/src/file/s3.service.ts index 7a103a7..b5231e0 100644 --- a/backend/src/file/s3.service.ts +++ b/backend/src/file/s3.service.ts @@ -25,6 +25,7 @@ import * as mime from "mime-types"; import { File } from "./file.service"; import { Readable } from "stream"; import { validate as isValidUUID } from "uuid"; +import * as archiver from "archiver"; @Injectable() export class S3FileService { @@ -275,7 +276,8 @@ export class S3FileService { } getS3Instance(): S3Client { - const checksumCalculation = this.config.get("s3.useChecksum") === true ? null : "WHEN_REQUIRED"; + const checksumCalculation = + this.config.get("s3.useChecksum") === true ? null : "WHEN_REQUIRED"; return new S3Client({ endpoint: this.config.get("s3.endpoint"), @@ -290,10 +292,95 @@ export class S3FileService { }); } - getZip() { - throw new BadRequestException( - "ZIP download is not supported with S3 storage", - ); + getZip(shareId: string) { + return new Promise(async (resolve, reject) => { + const s3Instance = this.getS3Instance(); + const bucketName = this.config.get("s3.bucketName"); + const compressionLevel = this.config.get("share.zipCompressionLevel"); + + const prefix = `${this.getS3Path()}${shareId}/`; + + try { + const listResponse = await s3Instance.send( + new ListObjectsV2Command({ + Bucket: bucketName, + Prefix: prefix, + }), + ); + + if (!listResponse.Contents || listResponse.Contents.length === 0) { + throw new NotFoundException(`No files found for share ${shareId}`); + } + + const archive = archiver("zip", { + zlib: { level: parseInt(compressionLevel) }, + }); + + archive.on("error", (err) => { + this.logger.error("Archive error", err); + reject(new InternalServerErrorException("Error creating ZIP file")); + }); + + const fileKeys = listResponse.Contents.filter( + (object) => object.Key && object.Key !== prefix, + ).map((object) => object.Key as string); + + if (fileKeys.length === 0) { + throw new NotFoundException( + `No valid files found for share ${shareId}`, + ); + } + + let filesAdded = 0; + + const processNextFile = async (index: number) => { + if (index >= fileKeys.length) { + archive.finalize(); + return; + } + + const key = fileKeys[index]; + const fileName = key.replace(prefix, ""); + + try { + const response = await s3Instance.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: key, + }), + ); + + if (response.Body instanceof Readable) { + const fileStream = response.Body; + + fileStream.on("end", () => { + filesAdded++; + processNextFile(index + 1); + }); + + fileStream.on("error", (err) => { + this.logger.error(`Error streaming file ${fileName}`, err); + processNextFile(index + 1); + }); + + archive.append(fileStream, { name: fileName }); + } else { + processNextFile(index + 1); + } + } catch (error) { + this.logger.error(`Error processing file ${fileName}`, error); + processNextFile(index + 1); + } + }; + + resolve(archive); + processNextFile(0); + } catch (error) { + this.logger.error("Error creating ZIP file", error); + + reject(new InternalServerErrorException("Error creating ZIP file")); + } + }); } getS3Path(): string { diff --git a/frontend/src/i18n/translations/en-US.ts b/frontend/src/i18n/translations/en-US.ts index a078d2e..266f560 100644 --- a/frontend/src/i18n/translations/en-US.ts +++ b/frontend/src/i18n/translations/en-US.ts @@ -366,6 +366,8 @@ export default { // /share/[id] "share.title": "Share {shareId}", "share.description": "Look what I've shared with you!", + "share.fileCount": + "{count, plural, =1 {# file} other {# files}} ยท {size} (zip file may be smaller due to compression)", "share.error.visitor-limit-exceeded.title": "Visitor limit exceeded", "share.error.visitor-limit-exceeded.description": "The visitor limit from this share has been exceeded.", @@ -408,14 +410,15 @@ export default { // /imprint "imprint.title": "Imprint", // END /imprint - + // /privacy "privacy.title": "Privacy Policy", // END /privacy // /admin/config "admin.config.config-file-warning.title": "Configuration file present", - "admin.config.config-file-warning.description": "As you have a configured Pingvin Share with a configuration file, you can't change the configuration through the UI.", + "admin.config.config-file-warning.description": + "As you have a configured Pingvin Share with a configuration file, you can't change the configuration through the UI.", "admin.config.title": "Configuration", "admin.config.category.general": "General", @@ -642,7 +645,8 @@ export default { "admin.config.category.s3": "S3", "admin.config.s3.enabled": "Enabled", - "admin.config.s3.enabled.description": "Whether S3 should be used to store the shared files instead of the local file system.", + "admin.config.s3.enabled.description": + "Whether S3 should be used to store the shared files instead of the local file system.", "admin.config.s3.endpoint": "Endpoint", "admin.config.s3.endpoint.description": "The URL of the S3 bucket.", "admin.config.s3.region": "Region", @@ -650,25 +654,34 @@ export default { "admin.config.s3.bucket-name": "Bucket name", "admin.config.s3.bucket-name.description": "The name of the S3 bucket.", "admin.config.s3.bucket-path": "Path", - "admin.config.s3.bucket-path.description": "The default path which should be used to store the files in the S3 bucket.", + "admin.config.s3.bucket-path.description": + "The default path which should be used to store the files in the S3 bucket.", "admin.config.s3.key": "Key", - "admin.config.s3.key.description": "The key which allows you to access the S3 bucket.", + "admin.config.s3.key.description": + "The key which allows you to access the S3 bucket.", "admin.config.s3.secret": "Secret", - "admin.config.s3.secret.description": "The secret which allows you to access the S3 bucket.", + "admin.config.s3.secret.description": + "The secret which allows you to access the S3 bucket.", "admin.config.s3.use-checksum": "Use checksum", - "admin.config.s3.use-checksum.description": "Turn off for backends that do not support checksum (e.g. B2).", + "admin.config.s3.use-checksum.description": + "Turn off for backends that do not support checksum (e.g. B2).", "admin.config.category.legal": "Legal", "admin.config.legal.enabled": "Enable legal notices", - "admin.config.legal.enabled.description": "Whether to show a link to imprint and privacy policy in the footer.", + "admin.config.legal.enabled.description": + "Whether to show a link to imprint and privacy policy in the footer.", "admin.config.legal.imprint-text": "Imprint text", - "admin.config.legal.imprint-text.description": "The text which should be shown in the imprint. Supports Markdown. Leave blank to link to an external imprint page.", + "admin.config.legal.imprint-text.description": + "The text which should be shown in the imprint. Supports Markdown. Leave blank to link to an external imprint page.", "admin.config.legal.imprint-url": "Imprint URL", - "admin.config.legal.imprint-url.description": "If you already have an imprint page you can link it here instead of using the text field.", + "admin.config.legal.imprint-url.description": + "If you already have an imprint page you can link it here instead of using the text field.", "admin.config.legal.privacy-policy-text": "Privacy policy text", - "admin.config.legal.privacy-policy-text.description": "The text which should be shown in the privacy policy. Supports Markdown. Leave blank to link to an external privacy policy page.", + "admin.config.legal.privacy-policy-text.description": + "The text which should be shown in the privacy policy. Supports Markdown. Leave blank to link to an external privacy policy page.", "admin.config.legal.privacy-policy-url": "Privacy policy URL", - "admin.config.legal.privacy-policy-url.description": "If you already have a privacy policy page you can link it here instead of using the text field.", + "admin.config.legal.privacy-policy-url.description": + "If you already have a privacy policy page you can link it here instead of using the text field.", // 404 "404.description": "Oops this page doesn't exist.", diff --git a/frontend/src/pages/share/[shareId]/index.tsx b/frontend/src/pages/share/[shareId]/index.tsx index 52d014d..a0b8bf1 100644 --- a/frontend/src/pages/share/[shareId]/index.tsx +++ b/frontend/src/pages/share/[shareId]/index.tsx @@ -2,6 +2,7 @@ import { Box, Group, Text, Title } from "@mantine/core"; import { useModals } from "@mantine/modals"; import { GetServerSidePropsContext } from "next"; import { useEffect, useState } from "react"; +import { FormattedMessage } from "react-intl"; import Meta from "../../../components/Meta"; import DownloadAllButton from "../../../components/share/DownloadAllButton"; import FileList from "../../../components/share/FileList"; @@ -11,6 +12,7 @@ import useTranslate from "../../../hooks/useTranslate.hook"; import shareService from "../../../services/share.service"; import { Share as ShareType } from "../../../types/share.type"; import toast from "../../../utils/toast.util"; +import { byteToHumanSizeString } from "../../../utils/fileSize.util"; export function getServerSideProps(context: GetServerSidePropsContext) { return { @@ -107,7 +109,25 @@ const Share = ({ shareId }: { shareId: string }) => { {share?.name || share?.id} {share?.description} + {share?.files?.length > 0 && ( + + + total + parseInt(file.size), + 0, + ) || 0, + ), + }} + /> + + )} + {share?.files.length > 1 && } diff --git a/package.json b/package.json index 398cc3e..ca16d29 100644 --- a/package.json +++ b/package.json @@ -11,5 +11,9 @@ }, "devDependencies": { "conventional-changelog-cli": "^3.0.0" + }, + "prettier": { + "singleQuote": false, + "trailingComma": "all" } }