From 6387127659906d0418ac062c4016f3b57a0dc964 Mon Sep 17 00:00:00 2001 From: SukantGujar Date: Tue, 19 Mar 2019 15:05:08 +0530 Subject: [PATCH] Add mongo content provider example. --- package.json | 5 +- src/createPartialContentHandler.ts | 3 +- .../express-mongo-server/MongoFileDocument.ts | 6 ++ src/examples/express-mongo-server/index.ts | 15 ++++ src/examples/express-mongo-server/logger.ts | 9 +++ .../mongoContentProvider.ts | 72 ++++++++++++++++++ src/examples/express-mongo-server/utils.ts | 17 +++++ yarn.lock | 75 ++++++++++++++++++- 8 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 src/examples/express-mongo-server/MongoFileDocument.ts create mode 100644 src/examples/express-mongo-server/index.ts create mode 100644 src/examples/express-mongo-server/logger.ts create mode 100644 src/examples/express-mongo-server/mongoContentProvider.ts create mode 100644 src/examples/express-mongo-server/utils.ts diff --git a/package.json b/package.json index ac51012d4..89579f647 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "build:watch": "npx tsc -w", "build:prod": "yarn clean && cross-env NODE_ENV=production tsc -p ./tsconfig.production.json", "push": "yarn test && yarn build:prod && yarn publish", - "run:examples:file": "node ./dist/examples/express-file-server/index.js" + "run:examples:file": "node ./dist/examples/express-file-server/index.js", + "run:examples:mongo": "node ./dist/examples/express-mongo-server/index.js" }, "keywords": [ "partial-content", @@ -23,11 +24,13 @@ "@types/chai": "^4.1.7", "@types/express": "^4.16.1", "@types/mocha": "^5.2.6", + "@types/mongodb": "^3.1.22", "@types/sinon": "^7.0.9", "chai": "^4.2.0", "cross-env": "^5.2.0", "express": "^4.16.4", "mocha": "^6.0.2", + "mongodb": "^3.1.13", "nyc": "^13.3.0", "rimraf": "^2.6.3", "sinon": "^7.2.7", diff --git a/src/createPartialContentHandler.ts b/src/createPartialContentHandler.ts index 5e6d6a3b2..86df530ca 100644 --- a/src/createPartialContentHandler.ts +++ b/src/createPartialContentHandler.ts @@ -54,6 +54,7 @@ export function createPartialContentHandler(contentProvider: ContentProvider, lo // Return 206 Partial Content status logger.debug("Returning partial content for range: ", JSON.stringify(range)); res.status(206); - getStream(range).pipe(res); + + return getStream(range).pipe(res); }; } diff --git a/src/examples/express-mongo-server/MongoFileDocument.ts b/src/examples/express-mongo-server/MongoFileDocument.ts new file mode 100644 index 000000000..f13e50cbb --- /dev/null +++ b/src/examples/express-mongo-server/MongoFileDocument.ts @@ -0,0 +1,6 @@ +export interface MongoFileDocument { + _id: string; + contentType: string; + filename: string; + size: number; +} diff --git a/src/examples/express-mongo-server/index.ts b/src/examples/express-mongo-server/index.ts new file mode 100644 index 000000000..a27a4e854 --- /dev/null +++ b/src/examples/express-mongo-server/index.ts @@ -0,0 +1,15 @@ +import express from "express"; +import { createPartialContentHandler } from "../../index"; +import { mongoContentProvider } from "./mongoContentProvider"; +import { logger } from "./logger"; + +const handler = createPartialContentHandler(mongoContentProvider, logger); + +const app = express(); +const port = 8080; + +app.get("/files/:name", handler); + +app.listen(port, () => { + logger.debug("Server started!"); +}); diff --git a/src/examples/express-mongo-server/logger.ts b/src/examples/express-mongo-server/logger.ts new file mode 100644 index 000000000..5dd53d659 --- /dev/null +++ b/src/examples/express-mongo-server/logger.ts @@ -0,0 +1,9 @@ +export const logger = { + debug(message: string, extra?: any) { + if (extra) { + console.log(`[debug]: ${message}`, extra); + } else { + console.log(`[debug]: ${message}`); + } + } +}; diff --git a/src/examples/express-mongo-server/mongoContentProvider.ts b/src/examples/express-mongo-server/mongoContentProvider.ts new file mode 100644 index 000000000..472451477 --- /dev/null +++ b/src/examples/express-mongo-server/mongoContentProvider.ts @@ -0,0 +1,72 @@ +import { Request } from "express"; +import fs from "fs"; +import path from "path"; +import { MongoClient, Db, GridFSBucket } from "mongodb"; +import { Range, ContentDoesNotExistError, ContentProvider } from "../../index"; +import { getFileMeta } from "./utils"; +import { logger } from "./logger"; +import { MongoFileDocument } from "./MongoFileDocument"; + +const connectionString = process.env["MongoUrl"]; + +if (!connectionString) { + throw new Error("MongoUrl env var is not defined!"); +} + +const client = new MongoClient(connectionString, { useNewUrlParser: true }); +let db: Db; +let bucket: GridFSBucket; +client.connect(error => { + if (error) { + throw error; + } + + db = client.db("test"); + + bucket = new GridFSBucket(db); + bucket.drop(error => { + if (error && error.message !== "ns not found") { + throw error; + } + + console.info("Dropped bucket."); + + const fileName = "readme.txt"; + + const readStream = fs.createReadStream(path.join(__dirname, "files", fileName)); + + readStream.on("close", () => { + console.log("readme.txt uploaded to gridfs."); + }); + + const writeStream = bucket.openUploadStream(fileName, { contentType: "text/plain" }); + + readStream.pipe(writeStream); + }); +}); + +export const mongoContentProvider: ContentProvider = async (req: Request) => { + const fileName = req.params.name; + const file = await getFileMeta(fileName, db); + if (!file) { + throw new ContentDoesNotExistError(`File doesn't exist: ${fileName}`); + } + const totalSize = file.size; + const mimeType = file.contentType; + const getStream = (range?: Range) => { + if (!range) { + return bucket.openDownloadStreamByName(fileName); + } + const { start, end } = range; + logger.debug(`start: ${start}, end: ${end}`); + // Note: The `end` offset in Mongo stream isn't inclusive, need to append 1 to it to ensure it returns + // the right sized content. Otherwise the stream size will be one byte short of the expected value. + return bucket.openDownloadStreamByName(fileName, { start, end: end + 1, revision: -1 }); + }; + return { + fileName, + totalSize, + mimeType, + getStream + }; +}; diff --git a/src/examples/express-mongo-server/utils.ts b/src/examples/express-mongo-server/utils.ts new file mode 100644 index 000000000..148361fdb --- /dev/null +++ b/src/examples/express-mongo-server/utils.ts @@ -0,0 +1,17 @@ +import { Db } from "mongodb"; +import { MongoFileDocument } from "./MongoFileDocument"; +import { logger } from "./logger"; + +export const getFileMeta = async (fileName: string, db: Db) => { + logger.debug(`Fetching file information for ${fileName}`); + const result = await db.collection("fs.files").findOne({ filename: fileName }); + logger.debug(`File meta: `, result); + return result + ? { + _id: result._id, + contentType: result.contentType, + fileName, + size: result.length + } + : null; +}; diff --git a/yarn.lock b/yarn.lock index 6d90e6164..3539a3e73 100644 --- a/yarn.lock +++ b/yarn.lock @@ -127,6 +127,13 @@ "@types/connect" "*" "@types/node" "*" +"@types/bson@*": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/bson/-/bson-4.0.0.tgz#9073772679d749116eb1dfca56f8eaac6d59cc7a" + integrity sha512-pq/rqJwJWkbS10crsG5bgnrisL8pML79KlMKQMoQwLUjlPAkrUHMvHJ3oGwE7WHR61Lv/nadMwXVAD2b+fpD8Q== + dependencies: + "@types/node" "*" + "@types/chai@^4.1.7": version "4.1.7" resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.1.7.tgz#1b8e33b61a8c09cbe1f85133071baa0dbf9fa71a" @@ -166,6 +173,14 @@ resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-5.2.6.tgz#b8622d50557dd155e9f2f634b7d68fd38de5e94b" integrity sha512-1axi39YdtBI7z957vdqXI4Ac25e7YihYQtJa+Clnxg1zTJEaIRbndt71O3sP4GAMgiAm0pY26/b9BrY4MR/PMw== +"@types/mongodb@^3.1.22": + version "3.1.22" + resolved "https://registry.yarnpkg.com/@types/mongodb/-/mongodb-3.1.22.tgz#66f96c7acbf7135ce6295a8d6c26f1d97388a84b" + integrity sha512-hvNR0txBlJJAy1eZOeIDshW4dnQaC694COou4eHHaMdIcteCfoCQATD7sYNlXxNxfTc1iIbHUi7A8CAhQe08uA== + dependencies: + "@types/bson" "*" + "@types/node" "*" + "@types/node@*": version "11.10.5" resolved "https://registry.yarnpkg.com/@types/node/-/node-11.10.5.tgz#fbaca34086bdc118011e1f05c47688d432f2d571" @@ -358,6 +373,11 @@ browser-stdout@1.3.1: resolved "https://registry.yarnpkg.com/browser-stdout/-/browser-stdout-1.3.1.tgz#baa559ee14ced73452229bad7326467c61fabd60" integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== +bson@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.1.tgz#4330f5e99104c4e751e7351859e2d408279f2f13" + integrity sha512-jCGVYLoYMHDkOsbwJZBCqwMHyH4c+wzgI9hG7Z6SZJRXWr+x58pdIbm2i9a/jFGCkRJqRUr8eoI7lDWa0hTkxg== + bytes@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" @@ -1491,6 +1511,11 @@ mem@^4.0.0: mimic-fn "^1.0.0" p-is-promise "^2.0.0" +memory-pager@^1.0.2: + version "1.5.0" + resolved "https://registry.yarnpkg.com/memory-pager/-/memory-pager-1.5.0.tgz#d8751655d22d384682741c972f2c3d6dfa3e66b5" + integrity sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg== + merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" @@ -1610,6 +1635,25 @@ mocha@^6.0.2: yargs-parser "11.1.1" yargs-unparser "1.5.0" +mongodb-core@3.1.11: + version "3.1.11" + resolved "https://registry.yarnpkg.com/mongodb-core/-/mongodb-core-3.1.11.tgz#b253038dbb4d7329f3d1c2ee5400bb0c9221fde5" + integrity sha512-rD2US2s5qk/ckbiiGFHeu+yKYDXdJ1G87F6CG3YdaZpzdOm5zpoAZd/EKbPmFO6cQZ+XVXBXBJ660sSI0gc6qg== + dependencies: + bson "^1.1.0" + require_optional "^1.0.1" + safe-buffer "^5.1.2" + optionalDependencies: + saslprep "^1.0.0" + +mongodb@^3.1.13: + version "3.1.13" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.1.13.tgz#f8cdcbb36ad7a08b570bd1271c8525753f75f9f4" + integrity sha512-sz2dhvBZQWf3LRNDhbd30KHVzdjZx9IKC0L+kSZ/gzYquCF5zPOgGqRz6sSCqYZtKP2ekB4nfLxhGtzGHnIKxA== + dependencies: + mongodb-core "3.1.11" + safe-buffer "^5.1.2" + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -2025,6 +2069,14 @@ require-main-filename@^1.0.1: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" integrity sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE= +require_optional@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/require_optional/-/require_optional-1.0.1.tgz#4cf35a4247f64ca3df8c2ef208cc494b1ca8fc2e" + integrity sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g== + dependencies: + resolve-from "^2.0.0" + semver "^5.1.0" + resolve-dir@^1.0.0, resolve-dir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-1.0.1.tgz#79a40644c362be82f26effe739c9bb5382046f43" @@ -2033,6 +2085,11 @@ resolve-dir@^1.0.0, resolve-dir@^1.0.1: expand-tilde "^2.0.0" global-modules "^1.0.0" +resolve-from@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-2.0.0.tgz#9480ab20e94ffa1d9e80a804c7ea147611966b57" + integrity sha1-lICrIOlP+h2egKgEx+oUdhGWa1c= + resolve-from@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" @@ -2062,7 +2119,7 @@ rimraf@^2.6.2, rimraf@^2.6.3: dependencies: glob "^7.1.3" -safe-buffer@5.1.2, safe-buffer@~5.1.1: +safe-buffer@5.1.2, safe-buffer@^5.1.2, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== @@ -2079,7 +2136,14 @@ safe-regex@^1.1.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0: +saslprep@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/saslprep/-/saslprep-1.0.2.tgz#da5ab936e6ea0bbae911ffec77534be370c9f52d" + integrity sha512-4cDsYuAjXssUSjxHKRe4DTZC0agDwsCqcMqtJAQPzC74nJ7LfAJflAtC1Zed5hMzEQKj82d3tuzqdGNRsLJ4Gw== + dependencies: + sparse-bitfield "^3.0.3" + +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.5.0, semver@^5.6.0: version "5.6.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.6.0.tgz#7e74256fbaa49c75aa7c7a205cc22799cac80004" integrity sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg== @@ -2229,6 +2293,13 @@ source-map@^0.6.1, source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== +sparse-bitfield@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz#ff4ae6e68656056ba4b3e792ab3334d38273ca11" + integrity sha1-/0rm5oZWBWuks+eSqzM004JzyhE= + dependencies: + memory-pager "^1.0.2" + spawn-wrap@^1.4.2: version "1.4.2" resolved "https://registry.yarnpkg.com/spawn-wrap/-/spawn-wrap-1.4.2.tgz#cff58e73a8224617b6561abdc32586ea0c82248c"