feat(s3): stream s3 content over a zip file (#822)

* chore: update package.json and enhance file handling in backend

* refactor: remove StreamResponseFilter and update getZip method for improved file handling

- Deleted StreamResponseFilter as it was no longer needed.
- Updated getZip method in LocalFileService to return a Promise and handle errors more effectively.
- Adjusted getZip method in FileService to await the storage service's getZip call.

* refactor: remove unused UseFilters import in file.controller.ts

backend/src/file/file.controller.ts

@@ -54,7 +54,7 @@ export class FileController {
     @Res({ passthrough: true }) res: Response,
     @Param("shareId") shareId: string,
   ) {
-    const zipStream = this.fileService.getZip(shareId);
+    const zipStream = await this.fileService.getZip(shareId);
 
     res.set({
       "Content-Type": "application/zip",

backend/src/file/file.service.ts

@@ -59,9 +59,9 @@ export class FileService {
     return storageService.deleteAllFiles(shareId);
   }
 
-  getZip(shareId: string) {
+  async getZip(shareId: string): Promise<Readable> {
     const storageService = this.getStorageService();
-    return storageService.getZip(shareId) as Readable;
+    return await storageService.getZip(shareId);
   }
 
   private async streamToUint8Array(stream: Readable): Promise<Uint8Array> {

backend/src/file/local.service.ts

@@ -14,6 +14,7 @@ import { ConfigService } from "src/config/config.service";
 import { PrismaService } from "src/prisma/prisma.service";
 import { validate as isValidUUID } from "uuid";
 import { SHARE_DIRECTORY } from "../constants";
+import { Readable } from "stream";
 
 @Injectable()
 export class LocalFileService {
@@ -155,7 +156,19 @@ export class LocalFileService {
     });
   }
 
-  getZip(shareId: string) {
+  async getZip(shareId: string): Promise<Readable> {
-    return createReadStream(`${SHARE_DIRECTORY}/${shareId}/archive.zip`);
+    return new Promise((resolve, reject) => {
+      const zipStream = createReadStream(
+        `${SHARE_DIRECTORY}/${shareId}/archive.zip`,
+      );
+
+      zipStream.on("error", (err) => {
+        reject(new InternalServerErrorException(err));
+      });
+
+      zipStream.on("open", () => {
+        resolve(zipStream);
+      });
+    });
   }
 }

backend/src/file/s3.service.ts

@@ -25,6 +25,7 @@ import * as mime from "mime-types";
 import { File } from "./file.service";
 import { Readable } from "stream";
 import { validate as isValidUUID } from "uuid";
+import * as archiver from "archiver";
 
 @Injectable()
 export class S3FileService {
@@ -275,7 +276,8 @@ export class S3FileService {
   }
 
   getS3Instance(): S3Client {
-    const checksumCalculation = this.config.get("s3.useChecksum") === true ? null : "WHEN_REQUIRED";
+    const checksumCalculation =
+      this.config.get("s3.useChecksum") === true ? null : "WHEN_REQUIRED";
 
     return new S3Client({
       endpoint: this.config.get("s3.endpoint"),
@@ -290,10 +292,95 @@ export class S3FileService {
     });
   }
 
-  getZip() {
+  getZip(shareId: string) {
-    throw new BadRequestException(
+    return new Promise<Readable>(async (resolve, reject) => {
-      "ZIP download is not supported with S3 storage",
+      const s3Instance = this.getS3Instance();
+      const bucketName = this.config.get("s3.bucketName");
+      const compressionLevel = this.config.get("share.zipCompressionLevel");
+
+      const prefix = `${this.getS3Path()}${shareId}/`;
+
+      try {
+        const listResponse = await s3Instance.send(
+          new ListObjectsV2Command({
+            Bucket: bucketName,
+            Prefix: prefix,
+          }),
         );
+
+        if (!listResponse.Contents || listResponse.Contents.length === 0) {
+          throw new NotFoundException(`No files found for share ${shareId}`);
+        }
+
+        const archive = archiver("zip", {
+          zlib: { level: parseInt(compressionLevel) },
+        });
+
+        archive.on("error", (err) => {
+          this.logger.error("Archive error", err);
+          reject(new InternalServerErrorException("Error creating ZIP file"));
+        });
+
+        const fileKeys = listResponse.Contents.filter(
+          (object) => object.Key && object.Key !== prefix,
+        ).map((object) => object.Key as string);
+
+        if (fileKeys.length === 0) {
+          throw new NotFoundException(
+            `No valid files found for share ${shareId}`,
+          );
+        }
+
+        let filesAdded = 0;
+
+        const processNextFile = async (index: number) => {
+          if (index >= fileKeys.length) {
+            archive.finalize();
+            return;
+          }
+
+          const key = fileKeys[index];
+          const fileName = key.replace(prefix, "");
+
+          try {
+            const response = await s3Instance.send(
+              new GetObjectCommand({
+                Bucket: bucketName,
+                Key: key,
+              }),
+            );
+
+            if (response.Body instanceof Readable) {
+              const fileStream = response.Body;
+
+              fileStream.on("end", () => {
+                filesAdded++;
+                processNextFile(index + 1);
+              });
+
+              fileStream.on("error", (err) => {
+                this.logger.error(`Error streaming file ${fileName}`, err);
+                processNextFile(index + 1);
+              });
+
+              archive.append(fileStream, { name: fileName });
+            } else {
+              processNextFile(index + 1);
+            }
+          } catch (error) {
+            this.logger.error(`Error processing file ${fileName}`, error);
+            processNextFile(index + 1);
+          }
+        };
+
+        resolve(archive);
+        processNextFile(0);
+      } catch (error) {
+        this.logger.error("Error creating ZIP file", error);
+
+        reject(new InternalServerErrorException("Error creating ZIP file"));
+      }
+    });
   }
 
   getS3Path(): string {

frontend/src/i18n/translations/en-US.ts

@@ -366,6 +366,8 @@ export default {
   // /share/[id]
   "share.title": "Share {shareId}",
   "share.description": "Look what I've shared with you!",
+  "share.fileCount":
+    "{count, plural, =1 {# file} other {# files}} · {size} (zip file may be smaller due to compression)",
   "share.error.visitor-limit-exceeded.title": "Visitor limit exceeded",
   "share.error.visitor-limit-exceeded.description":
     "The visitor limit from this share has been exceeded.",
@@ -415,7 +417,8 @@ export default {
 
   // /admin/config
   "admin.config.config-file-warning.title": "Configuration file present",
-  "admin.config.config-file-warning.description": "As you have a configured Pingvin Share with a configuration file, you can't change the configuration through the UI.",
+  "admin.config.config-file-warning.description":
+    "As you have a configured Pingvin Share with a configuration file, you can't change the configuration through the UI.",
 
   "admin.config.title": "Configuration",
   "admin.config.category.general": "General",
@@ -642,7 +645,8 @@ export default {
 
   "admin.config.category.s3": "S3",
   "admin.config.s3.enabled": "Enabled",
-  "admin.config.s3.enabled.description": "Whether S3 should be used to store the shared files instead of the local file system.",
+  "admin.config.s3.enabled.description":
+    "Whether S3 should be used to store the shared files instead of the local file system.",
   "admin.config.s3.endpoint": "Endpoint",
   "admin.config.s3.endpoint.description": "The URL of the S3 bucket.",
   "admin.config.s3.region": "Region",
@@ -650,25 +654,34 @@ export default {
   "admin.config.s3.bucket-name": "Bucket name",
   "admin.config.s3.bucket-name.description": "The name of the S3 bucket.",
   "admin.config.s3.bucket-path": "Path",
-  "admin.config.s3.bucket-path.description": "The default path which should be used to store the files in the S3 bucket.",
+  "admin.config.s3.bucket-path.description":
+    "The default path which should be used to store the files in the S3 bucket.",
   "admin.config.s3.key": "Key",
-  "admin.config.s3.key.description": "The key which allows you to access the S3 bucket.",
+  "admin.config.s3.key.description":
+    "The key which allows you to access the S3 bucket.",
   "admin.config.s3.secret": "Secret",
-  "admin.config.s3.secret.description": "The secret which allows you to access the S3 bucket.",
+  "admin.config.s3.secret.description":
+    "The secret which allows you to access the S3 bucket.",
   "admin.config.s3.use-checksum": "Use checksum",
-  "admin.config.s3.use-checksum.description": "Turn off for backends that do not support checksum (e.g. B2).",
+  "admin.config.s3.use-checksum.description":
+    "Turn off for backends that do not support checksum (e.g. B2).",
 
   "admin.config.category.legal": "Legal",
   "admin.config.legal.enabled": "Enable legal notices",
-  "admin.config.legal.enabled.description": "Whether to show a link to imprint and privacy policy in the footer.",
+  "admin.config.legal.enabled.description":
+    "Whether to show a link to imprint and privacy policy in the footer.",
   "admin.config.legal.imprint-text": "Imprint text",
-  "admin.config.legal.imprint-text.description": "The text which should be shown in the imprint. Supports Markdown. Leave blank to link to an external imprint page.",
+  "admin.config.legal.imprint-text.description":
+    "The text which should be shown in the imprint. Supports Markdown. Leave blank to link to an external imprint page.",
   "admin.config.legal.imprint-url": "Imprint URL",
-  "admin.config.legal.imprint-url.description": "If you already have an imprint page you can link it here instead of using the text field.",
+  "admin.config.legal.imprint-url.description":
+    "If you already have an imprint page you can link it here instead of using the text field.",
   "admin.config.legal.privacy-policy-text": "Privacy policy text",
-  "admin.config.legal.privacy-policy-text.description": "The text which should be shown in the privacy policy. Supports Markdown. Leave blank to link to an external privacy policy page.",
+  "admin.config.legal.privacy-policy-text.description":
+    "The text which should be shown in the privacy policy. Supports Markdown. Leave blank to link to an external privacy policy page.",
   "admin.config.legal.privacy-policy-url": "Privacy policy URL",
-  "admin.config.legal.privacy-policy-url.description": "If you already have a privacy policy page you can link it here instead of using the text field.",
+  "admin.config.legal.privacy-policy-url.description":
+    "If you already have a privacy policy page you can link it here instead of using the text field.",
 
   // 404
   "404.description": "Oops this page doesn't exist.",

frontend/src/pages/share/[shareId]/index.tsx

@@ -2,6 +2,7 @@ import { Box, Group, Text, Title } from "@mantine/core";
 import { useModals } from "@mantine/modals";
 import { GetServerSidePropsContext } from "next";
 import { useEffect, useState } from "react";
+import { FormattedMessage } from "react-intl";
 import Meta from "../../../components/Meta";
 import DownloadAllButton from "../../../components/share/DownloadAllButton";
 import FileList from "../../../components/share/FileList";
@@ -11,6 +12,7 @@ import useTranslate from "../../../hooks/useTranslate.hook";
 import shareService from "../../../services/share.service";
 import { Share as ShareType } from "../../../types/share.type";
 import toast from "../../../utils/toast.util";
+import { byteToHumanSizeString } from "../../../utils/fileSize.util";
 
 export function getServerSideProps(context: GetServerSidePropsContext) {
   return {
@@ -107,7 +109,25 @@ const Share = ({ shareId }: { shareId: string }) => {
         <Box style={{ maxWidth: "70%" }}>
           <Title order={3}>{share?.name || share?.id}</Title>
           <Text size="sm">{share?.description}</Text>
+          {share?.files?.length > 0 && (
+            <Text size="sm" color="dimmed" mt={5}>
+              <FormattedMessage
+                id="share.fileCount"
+                values={{
+                  count: share?.files?.length || 0,
+                  size: byteToHumanSizeString(
+                    share?.files?.reduce(
+                      (total: number, file: { size: string }) =>
+                        total + parseInt(file.size),
+                      0,
+                    ) || 0,
+                  ),
+                }}
+              />
+            </Text>
+          )}
         </Box>
+
         {share?.files.length > 1 && <DownloadAllButton shareId={shareId} />}
       </Group>
 

package.json

@@ -11,5 +11,9 @@
   },
   "devDependencies": {
     "conventional-changelog-cli": "^3.0.0"
+  },
+  "prettier": {
+    "singleQuote": false,
+    "trailingComma": "all"
   }
 }