swissmakers_gmbh/swiss-datashare
12
0
Fork 0
mirror of https://github.com/swissmakers/swiss-datashare.git synced 2026-03-26 11:13:25 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(s3): stream s3 content over a zip file (#822)

Browse Source 
* chore: update package.json and enhance file handling in backend

* refactor: remove StreamResponseFilter and update getZip method for improved file handling

- Deleted StreamResponseFilter as it was no longer needed.
- Updated getZip method in LocalFileService to return a Promise and handle errors more effectively.
- Adjusted getZip method in FileService to await the storage service's getZip call.

* refactor: remove unused UseFilters import in file.controller.ts
This commit is contained in:
Lucas Reis
2025-05-04 20:37:04 +02:00
committed by GitHub
parent 853f217bf1
commit ccc783ab6a
7 changed files with 159 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/file/file.controller.ts
View File

@@ -54,7 +54,7 @@ export class FileController {
@Res({ passthrough: true }) res: Response,
@Param("shareId") shareId: string,
) {
const zipStream = this.fileService.getZip(shareId);
const zipStream = await this.fileService.getZip(shareId);
res.set({
"Content-Type": "application/zip",

4
backend/src/file/file.service.ts
View File

@@ -59,9 +59,9 @@ export class FileService {
return storageService.deleteAllFiles(shareId);
}
getZip(shareId: string) {
async getZip(shareId: string): Promise<Readable> {
const storageService = this.getStorageService();
return storageService.getZip(shareId) as Readable;
return await storageService.getZip(shareId);
}
private async streamToUint8Array(stream: Readable): Promise<Uint8Array> {

17
backend/src/file/local.service.ts
View File

@@ -14,6 +14,7 @@ import { ConfigService } from "src/config/config.service";
import { PrismaService } from "src/prisma/prisma.service";
import { validate as isValidUUID } from "uuid";
import { SHARE_DIRECTORY } from "../constants";
import { Readable } from "stream";
@Injectable()
export class LocalFileService {
@@ -155,7 +156,19 @@ export class LocalFileService {
});
}
getZip(shareId: string) {
return createReadStream(`${SHARE_DIRECTORY}/${shareId}/archive.zip`);
async getZip(shareId: string): Promise<Readable> {
return new Promise((resolve, reject) => {
const zipStream = createReadStream(
`${SHARE_DIRECTORY}/${shareId}/archive.zip`,
);
zipStream.on("error", (err) => {
reject(new InternalServerErrorException(err));
});
zipStream.on("open", () => {
resolve(zipStream);
});
});
}
}

97
backend/src/file/s3.service.ts
View File

@@ -25,6 +25,7 @@ import * as mime from "mime-types";
import { File } from "./file.service";
import { Readable } from "stream";
import { validate as isValidUUID } from "uuid";
import * as archiver from "archiver";
@Injectable()
export class S3FileService {
@@ -275,7 +276,8 @@ export class S3FileService {
}
getS3Instance(): S3Client {
const checksumCalculation = this.config.get("s3.useChecksum") === true ? null : "WHEN_REQUIRED";
const checksumCalculation =
this.config.get("s3.useChecksum") === true ? null : "WHEN_REQUIRED";
return new S3Client({
endpoint: this.config.get("s3.endpoint"),
@@ -290,10 +292,95 @@ export class S3FileService {
});
}
getZip() {
throw new BadRequestException(
"ZIP download is not supported with S3 storage",
);
getZip(shareId: string) {
return new Promise<Readable>(async (resolve, reject) => {
const s3Instance = this.getS3Instance();
const bucketName = this.config.get("s3.bucketName");
const compressionLevel = this.config.get("share.zipCompressionLevel");
const prefix = `${this.getS3Path()}${shareId}/`;
try {
const listResponse = await s3Instance.send(
new ListObjectsV2Command({
Bucket: bucketName,
Prefix: prefix,
}),
);
if (!listResponse.Contents || listResponse.Contents.length === 0) {
throw new NotFoundException(`No files found for share ${shareId}`);
}
const archive = archiver("zip", {
zlib: { level: parseInt(compressionLevel) },
});
archive.on("error", (err) => {
this.logger.error("Archive error", err);
reject(new InternalServerErrorException("Error creating ZIP file"));
});
const fileKeys = listResponse.Contents.filter(
(object) => object.Key && object.Key !== prefix,
).map((object) => object.Key as string);
if (fileKeys.length === 0) {
throw new NotFoundException(
`No valid files found for share ${shareId}`,
);
}
let filesAdded = 0;
const processNextFile = async (index: number) => {
if (index >= fileKeys.length) {
archive.finalize();
return;
}
const key = fileKeys[index];
const fileName = key.replace(prefix, "");
try {
const response = await s3Instance.send(
new GetObjectCommand({
Bucket: bucketName,
Key: key,
}),
);
if (response.Body instanceof Readable) {
const fileStream = response.Body;
fileStream.on("end", () => {
filesAdded++;
processNextFile(index + 1);
});
fileStream.on("error", (err) => {
this.logger.error(`Error streaming file ${fileName}`, err);
processNextFile(index + 1);
});
archive.append(fileStream, { name: fileName });
} else {
processNextFile(index + 1);
}
} catch (error) {
this.logger.error(`Error processing file ${fileName}`, error);
processNextFile(index + 1);
}
};
resolve(archive);
processNextFile(0);
} catch (error) {
this.logger.error("Error creating ZIP file", error);
reject(new InternalServerErrorException("Error creating ZIP file"));
}
});
}
getS3Path(): string {

37
frontend/src/i18n/translations/en-US.ts
View File

@@ -366,6 +366,8 @@ export default {
// /share/[id]
"share.title": "Share {shareId}",
"share.description": "Look what I've shared with you!",
"share.fileCount":
"{count, plural, =1 {# file} other {# files}} · {size} (zip file may be smaller due to compression)",
"share.error.visitor-limit-exceeded.title": "Visitor limit exceeded",
"share.error.visitor-limit-exceeded.description":
"The visitor limit from this share has been exceeded.",
@@ -408,14 +410,15 @@ export default {
// /imprint
"imprint.title": "Imprint",
// END /imprint
// /privacy
"privacy.title": "Privacy Policy",
// END /privacy
// /admin/config
"admin.config.config-file-warning.title": "Configuration file present",
"admin.config.config-file-warning.description": "As you have a configured Pingvin Share with a configuration file, you can't change the configuration through the UI.",
"admin.config.config-file-warning.description":
"As you have a configured Pingvin Share with a configuration file, you can't change the configuration through the UI.",
"admin.config.title": "Configuration",
"admin.config.category.general": "General",
@@ -642,7 +645,8 @@ export default {
"admin.config.category.s3": "S3",
"admin.config.s3.enabled": "Enabled",
"admin.config.s3.enabled.description": "Whether S3 should be used to store the shared files instead of the local file system.",
"admin.config.s3.enabled.description":
"Whether S3 should be used to store the shared files instead of the local file system.",
"admin.config.s3.endpoint": "Endpoint",
"admin.config.s3.endpoint.description": "The URL of the S3 bucket.",
"admin.config.s3.region": "Region",
@@ -650,25 +654,34 @@ export default {
"admin.config.s3.bucket-name": "Bucket name",
"admin.config.s3.bucket-name.description": "The name of the S3 bucket.",
"admin.config.s3.bucket-path": "Path",
"admin.config.s3.bucket-path.description": "The default path which should be used to store the files in the S3 bucket.",
"admin.config.s3.bucket-path.description":
"The default path which should be used to store the files in the S3 bucket.",
"admin.config.s3.key": "Key",
"admin.config.s3.key.description": "The key which allows you to access the S3 bucket.",
"admin.config.s3.key.description":
"The key which allows you to access the S3 bucket.",
"admin.config.s3.secret": "Secret",
"admin.config.s3.secret.description": "The secret which allows you to access the S3 bucket.",
"admin.config.s3.secret.description":
"The secret which allows you to access the S3 bucket.",
"admin.config.s3.use-checksum": "Use checksum",
"admin.config.s3.use-checksum.description": "Turn off for backends that do not support checksum (e.g. B2).",
"admin.config.s3.use-checksum.description":
"Turn off for backends that do not support checksum (e.g. B2).",
"admin.config.category.legal": "Legal",
"admin.config.legal.enabled": "Enable legal notices",
"admin.config.legal.enabled.description": "Whether to show a link to imprint and privacy policy in the footer.",
"admin.config.legal.enabled.description":
"Whether to show a link to imprint and privacy policy in the footer.",
"admin.config.legal.imprint-text": "Imprint text",
"admin.config.legal.imprint-text.description": "The text which should be shown in the imprint. Supports Markdown. Leave blank to link to an external imprint page.",
"admin.config.legal.imprint-text.description":
"The text which should be shown in the imprint. Supports Markdown. Leave blank to link to an external imprint page.",
"admin.config.legal.imprint-url": "Imprint URL",
"admin.config.legal.imprint-url.description": "If you already have an imprint page you can link it here instead of using the text field.",
"admin.config.legal.imprint-url.description":
"If you already have an imprint page you can link it here instead of using the text field.",
"admin.config.legal.privacy-policy-text": "Privacy policy text",
"admin.config.legal.privacy-policy-text.description": "The text which should be shown in the privacy policy. Supports Markdown. Leave blank to link to an external privacy policy page.",
"admin.config.legal.privacy-policy-text.description":
"The text which should be shown in the privacy policy. Supports Markdown. Leave blank to link to an external privacy policy page.",
"admin.config.legal.privacy-policy-url": "Privacy policy URL",
"admin.config.legal.privacy-policy-url.description": "If you already have a privacy policy page you can link it here instead of using the text field.",
"admin.config.legal.privacy-policy-url.description":
"If you already have a privacy policy page you can link it here instead of using the text field.",
// 404
"404.description": "Oops this page doesn't exist.",

20
frontend/src/pages/share/[shareId]/index.tsx
View File

@@ -2,6 +2,7 @@ import { Box, Group, Text, Title } from "@mantine/core";
import { useModals } from "@mantine/modals";
import { GetServerSidePropsContext } from "next";
import { useEffect, useState } from "react";
import { FormattedMessage } from "react-intl";
import Meta from "../../../components/Meta";
import DownloadAllButton from "../../../components/share/DownloadAllButton";
import FileList from "../../../components/share/FileList";
@@ -11,6 +12,7 @@ import useTranslate from "../../../hooks/useTranslate.hook";
import shareService from "../../../services/share.service";
import { Share as ShareType } from "../../../types/share.type";
import toast from "../../../utils/toast.util";
import { byteToHumanSizeString } from "../../../utils/fileSize.util";
export function getServerSideProps(context: GetServerSidePropsContext) {
return {
@@ -107,7 +109,25 @@ const Share = ({ shareId }: { shareId: string }) => {
<Box style={{ maxWidth: "70%" }}>
<Title order={3}>{share?.name || share?.id}</Title>
<Text size="sm">{share?.description}</Text>
{share?.files?.length > 0 && (
<Text size="sm" color="dimmed" mt={5}>
<FormattedMessage
id="share.fileCount"
values={{
count: share?.files?.length || 0,
size: byteToHumanSizeString(
share?.files?.reduce(
(total: number, file: { size: string }) =>
total + parseInt(file.size),
0,
) || 0,
),
}}
/>
</Text>
)}
</Box>
{share?.files.length > 1 && <DownloadAllButton shareId={shareId} />}
</Group>

4
package.json
View File

@@ -11,5 +11,9 @@
},
"devDependencies": {
"conventional-changelog-cli": "^3.0.0"
},
"prettier": {
"singleQuote": false,
"trailingComma": "all"
}
}