initial commit

node_modules/async-sema/lib/index.d.ts

@@ -0,0 +1,25 @@
+export declare class Sema {
+    private nrTokens;
+    private free;
+    private waiting;
+    private releaseEmitter;
+    private noTokens;
+    private pauseFn?;
+    private resumeFn?;
+    private paused;
+    constructor(nr: number, { initFn, pauseFn, resumeFn, capacity, }?: {
+        initFn?: () => any;
+        pauseFn?: () => void;
+        resumeFn?: () => void;
+        capacity?: number;
+    });
+    tryAcquire(): any | undefined;
+    acquire(): Promise<any>;
+    release(token?: any): void;
+    drain(): Promise<any[]>;
+    nrWaiting(): number;
+}
+export declare function RateLimit(rps: number, { timeUnit, uniformDistribution, }?: {
+    timeUnit?: number;
+    uniformDistribution?: boolean;
+}): () => Promise<void>;

node_modules/async-sema/lib/index.js

@@ -0,0 +1,163 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RateLimit = exports.Sema = void 0;
+const events_1 = __importDefault(require("events"));
+function arrayMove(src, srcIndex, dst, dstIndex, len) {
+    for (let j = 0; j < len; ++j) {
+        dst[j + dstIndex] = src[j + srcIndex];
+        src[j + srcIndex] = void 0;
+    }
+}
+function pow2AtLeast(n) {
+    n = n >>> 0;
+    n = n - 1;
+    n = n | (n >> 1);
+    n = n | (n >> 2);
+    n = n | (n >> 4);
+    n = n | (n >> 8);
+    n = n | (n >> 16);
+    return n + 1;
+}
+function getCapacity(capacity) {
+    return pow2AtLeast(Math.min(Math.max(16, capacity), 1073741824));
+}
+// Deque is based on https://github.com/petkaantonov/deque/blob/master/js/deque.js
+// Released under the MIT License: https://github.com/petkaantonov/deque/blob/6ef4b6400ad3ba82853fdcc6531a38eb4f78c18c/LICENSE
+class Deque {
+    constructor(capacity) {
+        this._capacity = getCapacity(capacity);
+        this._length = 0;
+        this._front = 0;
+        this.arr = [];
+    }
+    push(item) {
+        const length = this._length;
+        this.checkCapacity(length + 1);
+        const i = (this._front + length) & (this._capacity - 1);
+        this.arr[i] = item;
+        this._length = length + 1;
+        return length + 1;
+    }
+    pop() {
+        const length = this._length;
+        if (length === 0) {
+            return void 0;
+        }
+        const i = (this._front + length - 1) & (this._capacity - 1);
+        const ret = this.arr[i];
+        this.arr[i] = void 0;
+        this._length = length - 1;
+        return ret;
+    }
+    shift() {
+        const length = this._length;
+        if (length === 0) {
+            return void 0;
+        }
+        const front = this._front;
+        const ret = this.arr[front];
+        this.arr[front] = void 0;
+        this._front = (front + 1) & (this._capacity - 1);
+        this._length = length - 1;
+        return ret;
+    }
+    get length() {
+        return this._length;
+    }
+    checkCapacity(size) {
+        if (this._capacity < size) {
+            this.resizeTo(getCapacity(this._capacity * 1.5 + 16));
+        }
+    }
+    resizeTo(capacity) {
+        const oldCapacity = this._capacity;
+        this._capacity = capacity;
+        const front = this._front;
+        const length = this._length;
+        if (front + length > oldCapacity) {
+            const moveItemsCount = (front + length) & (oldCapacity - 1);
+            arrayMove(this.arr, 0, this.arr, oldCapacity, moveItemsCount);
+        }
+    }
+}
+class ReleaseEmitter extends events_1.default {
+}
+function isFn(x) {
+    return typeof x === 'function';
+}
+function defaultInit() {
+    return '1';
+}
+class Sema {
+    constructor(nr, { initFn = defaultInit, pauseFn, resumeFn, capacity = 10, } = {}) {
+        if (isFn(pauseFn) !== isFn(resumeFn)) {
+            throw new Error('pauseFn and resumeFn must be both set for pausing');
+        }
+        this.nrTokens = nr;
+        this.free = new Deque(nr);
+        this.waiting = new Deque(capacity);
+        this.releaseEmitter = new ReleaseEmitter();
+        this.noTokens = initFn === defaultInit;
+        this.pauseFn = pauseFn;
+        this.resumeFn = resumeFn;
+        this.paused = false;
+        this.releaseEmitter.on('release', (token) => {
+            const p = this.waiting.shift();
+            if (p) {
+                p.resolve(token);
+            }
+            else {
+                if (this.resumeFn && this.paused) {
+                    this.paused = false;
+                    this.resumeFn();
+                }
+                this.free.push(token);
+            }
+        });
+        for (let i = 0; i < nr; i++) {
+            this.free.push(initFn());
+        }
+    }
+    tryAcquire() {
+        return this.free.pop();
+    }
+    async acquire() {
+        let token = this.tryAcquire();
+        if (token !== void 0) {
+            return token;
+        }
+        return new Promise((resolve, reject) => {
+            if (this.pauseFn && !this.paused) {
+                this.paused = true;
+                this.pauseFn();
+            }
+            this.waiting.push({ resolve, reject });
+        });
+    }
+    release(token) {
+        this.releaseEmitter.emit('release', this.noTokens ? '1' : token);
+    }
+    drain() {
+        const a = new Array(this.nrTokens);
+        for (let i = 0; i < this.nrTokens; i++) {
+            a[i] = this.acquire();
+        }
+        return Promise.all(a);
+    }
+    nrWaiting() {
+        return this.waiting.length;
+    }
+}
+exports.Sema = Sema;
+function RateLimit(rps, { timeUnit = 1000, uniformDistribution = false, } = {}) {
+    const sema = new Sema(uniformDistribution ? 1 : rps);
+    const delay = uniformDistribution ? timeUnit / rps : timeUnit;
+    return async function rl() {
+        await sema.acquire();
+        setTimeout(() => sema.release(), delay);
+    };
+}
+exports.RateLimit = RateLimit;

node_modules/async-sema/license.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Vercel, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

node_modules/async-sema/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "async-sema",
+  "version": "3.1.1",
+  "description": "Semaphore using `async` and `await`",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vercel/async-sema.git"
+  },
+  "author": "Olli Vanhoja",
+  "keywords": [
+    "semaphore",
+    "async",
+    "await"
+  ],
+  "homepage": "https://github.com/vercel/async-sema",
+  "bugs": {
+    "url": "https://github.com/vercel/async-sema/issues"
+  },
+  "license": "MIT",
+  "files": [
+    "lib"
+  ],
+  "main": "lib/index.js",
+  "scripts": {
+    "build": "tsc",
+    "lint:staged": "lint-staged",
+    "prepublishOnly": "yarn build",
+    "prettier": "prettier --write --single-quote './{src,test}/**/*.ts'",
+    "test": "jest"
+  },
+  "devDependencies": {
+    "@types/jest": "27.0.1",
+    "@types/node": "16.6.1",
+    "jest": "27.0.6",
+    "lint-staged": "11.1.2",
+    "pre-commit": "1.2.2",
+    "prettier": "2.3.2",
+    "ts-jest": "27.0.4",
+    "typescript": "4.3.5"
+  },
+  "pre-commit": "lint:staged",
+  "lint-staged": {
+    "*.{js,ts}": [
+      "prettier --write --single-quote",
+      "git add"
+    ]
+  },
+  "jest": {
+    "preset": "ts-jest",
+    "verbose": false,
+    "testURL": "http://localhost/"
+  }
+}

node_modules/async-sema/readme.md

@@ -0,0 +1,157 @@
+# async-sema
+
+This is a semaphore implementation for use with `async` and `await`. The
+implementation follows the traditional definition of a semaphore rather than the
+definition of an asynchronous semaphore seen in some js community examples.
+Where as the latter one generally allows every defined task to proceed
+immediately and synchronizes at the end, async-sema allows only a selected
+number of tasks to proceed at once while the rest will remain waiting.
+
+Async-sema manages the semaphore count as a list of tokens instead of a single
+variable containing the number of available resources. This enables an
+interesting application of managing the actual resources with the semaphore
+object itself. To make it practical the constructor for Sema includes an option
+for providing an init function for the semaphore tokens. Use of a custom token
+initializer is demonstrated in `examples/pooling.js`.
+
+## Usage
+
+Firstly, add the package to your project's `dependencies`:
+
+```bash
+npm install --save async-sema
+```
+
+or
+
+```bash
+yarn add async-sema
+```
+
+Then start using it like shown in the following example. Check more
+use case examples [here](./examples).
+
+## Example
+
+```js
+const { Sema } = require('async-sema');
+const s = new Sema(
+  4, // Allow 4 concurrent async calls
+  {
+    capacity: 100 // Prealloc space for 100 tokens
+  }
+);
+
+async function fetchData(x) {
+  await s.acquire()
+  try {
+    console.log(s.nrWaiting() + ' calls to fetch are waiting')
+    // ... do some async stuff with x
+  } finally {
+    s.release();
+  }
+}
+
+const data = await Promise.all(array.map(fetchData));
+```
+
+The package also offers a simple rate limiter utilizing the semaphore
+implementation.
+
+```js
+const { RateLimit } = require('async-sema');
+
+async function f() {
+  const lim = RateLimit(5); // rps
+
+  for (let i = 0; i < n; i++) {
+    await lim();
+    // ... do something async
+  }
+}
+```
+
+## API
+
+### Sema
+
+#### Constructor(nr, { initFn, pauseFn, resumeFn, capacity })
+
+Creates a semaphore object. The first argument is mandatory and the second
+argument is optional.
+
+- `nr` The maximum number of callers allowed to acquire the semaphore
+  concurrently.
+- `initFn` Function that is used to initialize the tokens used to manage
+  the semaphore. The default is `() => '1'`.
+- `pauseFn` An optional fuction that is called to opportunistically request
+  pausing the the incoming stream of data, instead of piling up waiting
+  promises and possibly running out of memory.
+  See [examples/pausing.js](./examples/pausing.js).
+- `resumeFn` An optional function that is called when there is room again
+  to accept new waiters on the semaphore. This function must be declared
+  if a `pauseFn` is declared.
+- `capacity` Sets the size of the preallocated waiting list inside the
+  semaphore. This is typically used by high performance where the developer
+  can make a rough estimate of the number of concurrent users of a semaphore.
+
+#### async drain()
+
+Drains the semaphore and returns all the initialized tokens in an array.
+Draining is an ideal way to ensure there are no pending async tasks, for
+example before a process will terminate.
+
+#### nrWaiting()
+
+Returns the number of callers waiting on the semaphore, i.e. the number of
+pending promises.
+
+#### tryAcquire()
+
+Attempt to acquire a token from the semaphore, if one is available immediately.
+Otherwise, return `undefined`.
+
+#### async acquire()
+
+Acquire a token from the semaphore, thus decrement the number of available
+execution slots. If `initFn` is not used then the return value of the function
+can be discarded.
+
+#### release(token)
+
+Release the semaphore, thus increment the number of free execution slots. If
+`initFn` is used then the `token` returned by `acquire()` should be given as
+an argument when calling this function.
+
+### RateLimit(rps, { timeUnit, uniformDistribution })
+
+Creates a rate limiter function that blocks with a promise whenever the rate
+limit is hit and resolves the promise once the call rate is within the limit
+set by `rps`. The second argument is optional.
+
+The `timeUnit` is an optional argument setting the width of the rate limiting
+window in milliseconds. The default `timeUnit` is `1000 ms`, therefore making
+the `rps` argument act as requests per second limit.
+
+The `uniformDistribution` argument enforces a discrete uniform distribution over
+time, instead of the default that allows hitting the function `rps` time and
+then pausing for `timeWindow` milliseconds. Setting the `uniformDistribution`
+option is mainly useful in a situation where the flow of rate limit function
+calls is continuous and and occuring faster than `timeUnit` (e.g. reading a
+file) and not enabling it would cause the maximum number of calls to resolve
+immediately (thus exhaust the limit immediately) and therefore the next bunch
+calls would need to wait for `timeWindow` milliseconds. However if the flow is
+sparse then this option may make the
+code run slower with no advantages.
+
+## Contributing
+
+1. [Fork](https://help.github.com/articles/fork-a-repo/) this repository to your own GitHub account and then [clone](https://help.github.com/articles/cloning-a-repository/) it to your local device
+2. Move into the directory of the clone: `cd async-sema`
+3. Link it to the global module directory of Node.js: `npm link`
+
+Inside the project where you want to test your clone of the package, you can now either use `npm link async-sema` to link the clone to the local dependencies.
+
+## Author
+
+Olli Vanhoja ([@OVanhoja](https://twitter.com/OVanhoja))