node_modules ignore

This commit is contained in:
2025-05-08 23:43:47 +02:00
parent e19d52f172
commit 4574544c9f
65041 changed files with 10593536 additions and 0 deletions

View File

@@ -0,0 +1,5 @@
export * from './local-destination';
export * from './local-source';
export * from './remote-destination';
export * from './remote-source';
//# sourceMappingURL=index.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/strapi/providers/index.ts"],"names":[],"mappings":"AACA,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAG/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC"}

View File

@@ -0,0 +1,16 @@
'use strict';
var index = require('./local-destination/index.js');
var index$1 = require('./local-source/index.js');
var index$2 = require('./remote-destination/index.js');
var index$3 = require('./remote-source/index.js');
// Local
exports.DEFAULT_CONFLICT_STRATEGY = index.DEFAULT_CONFLICT_STRATEGY;
exports.VALID_CONFLICT_STRATEGIES = index.VALID_CONFLICT_STRATEGIES;
exports.createLocalStrapiDestinationProvider = index.createLocalStrapiDestinationProvider;
exports.createLocalStrapiSourceProvider = index$1.createLocalStrapiSourceProvider;
exports.createRemoteStrapiDestinationProvider = index$2.createRemoteStrapiDestinationProvider;
exports.createRemoteStrapiSourceProvider = index$3.createRemoteStrapiSourceProvider;
//# sourceMappingURL=index.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.js","sources":["../../../src/strapi/providers/index.ts"],"sourcesContent":["// Local\nexport * from './local-destination';\nexport * from './local-source';\n\n// Remote\nexport * from './remote-destination';\nexport * from './remote-source';\n"],"names":[],"mappings":";;;;;;;AAAA;;;;;;;;;"}

View File

@@ -0,0 +1,7 @@
export { DEFAULT_CONFLICT_STRATEGY, VALID_CONFLICT_STRATEGIES, createLocalStrapiDestinationProvider } from './local-destination/index.mjs';
export { createLocalStrapiSourceProvider } from './local-source/index.mjs';
export { createRemoteStrapiDestinationProvider } from './remote-destination/index.mjs';
export { createRemoteStrapiSourceProvider } from './remote-source/index.mjs';
// Local
//# sourceMappingURL=index.mjs.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.mjs","sources":["../../../src/strapi/providers/index.ts"],"sourcesContent":["// Local\nexport * from './local-destination';\nexport * from './local-source';\n\n// Remote\nexport * from './remote-destination';\nexport * from './remote-source';\n"],"names":[],"mappings":";;;;;AAAA"}

View File

@@ -0,0 +1,38 @@
/// <reference types="node" />
import { Writable } from 'stream';
import type { Core, Struct } from '@strapi/types';
import type { IDestinationProvider, IMetadata, ProviderType, Transaction } from '../../../../types';
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
import { restore } from './strategies';
export declare const VALID_CONFLICT_STRATEGIES: string[];
export declare const DEFAULT_CONFLICT_STRATEGY = "restore";
export interface ILocalStrapiDestinationProviderOptions {
getStrapi(): Core.Strapi | Promise<Core.Strapi>;
autoDestroy?: boolean;
restore?: restore.IRestoreOptions;
strategy: 'restore';
}
declare class LocalStrapiDestinationProvider implements IDestinationProvider {
#private;
name: string;
type: ProviderType;
options: ILocalStrapiDestinationProviderOptions;
strapi?: Core.Strapi;
transaction?: Transaction;
uploadsBackupDirectoryName: string;
onWarning?: ((message: string) => void) | undefined;
constructor(options: ILocalStrapiDestinationProviderOptions);
bootstrap(diagnostics?: IDiagnosticReporter): Promise<void>;
close(): Promise<void>;
rollback(): Promise<void>;
beforeTransfer(): Promise<void>;
getMetadata(): IMetadata;
getSchemas(): Record<string, Struct.Schema>;
createEntitiesWriteStream(): Writable;
createAssetsWriteStream(): Promise<Writable>;
createConfigurationWriteStream(): Promise<Writable>;
createLinksWriteStream(): Promise<Writable>;
}
export declare const createLocalStrapiDestinationProvider: (options: ILocalStrapiDestinationProviderOptions) => LocalStrapiDestinationProvider;
export {};
//# sourceMappingURL=index.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/strapi/providers/local-destination/index.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,QAAQ,EAAY,MAAM,QAAQ,CAAC;AAI5C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,KAAK,EAEV,oBAAoB,EAEpB,SAAS,EACT,YAAY,EACZ,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAErE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AASvC,eAAO,MAAM,yBAAyB,UAAc,CAAC;AACrD,eAAO,MAAM,yBAAyB,YAAY,CAAC;AAEnD,MAAM,WAAW,sCAAsC;IACrD,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC;IAClC,QAAQ,EAAE,SAAS,CAAC;CACrB;AAED,cAAM,8BAA+B,YAAW,oBAAoB;;IAClE,IAAI,SAA+B;IAEnC,IAAI,EAAE,YAAY,CAAiB;IAEnC,OAAO,EAAE,sCAAsC,CAAC;IAEhD,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;IAErB,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,0BAA0B,EAAE,MAAM,CAAC;IAEnC,SAAS,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;gBASxC,OAAO,EAAE,sCAAsC;IAMrD,SAAS,CAAC,WAAW,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsC3D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmEtB,QAAQ;IAMR,cAAc;IAkBpB,WAAW,IAAI,SAAS;IAcxB,UAAU,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;IAY3C,yBAAyB,IAAI,QAAQ;IA2F/B,uBAAuB,IAAI,OAAO,CAAC,QAAQ,CAAC;IA2F5C,8BAA8B,IAAI,OAAO,CAAC,QAAQ,CAAC;IAgBnD,sBAAsB,IAAI,OAAO,CAAC,QAAQ,CAAC;CAmBlD;AAED,eAAO,MAAM,oCAAoC,YACtC,sCAAsC,mCAGhD,CAAC"}

View File

@@ -0,0 +1,419 @@
'use strict';
var stream = require('stream');
var path = require('path');
var fse = require('fs-extra');
var index = require('./strategies/restore/index.js');
require('crypto');
require('lodash/fp');
var schema = require('../../../utils/schema.js');
var transaction = require('../../../utils/transaction.js');
require('events');
var providers = require('../../../errors/providers.js');
var providers$1 = require('../../../utils/providers.js');
var entities = require('./strategies/restore/entities.js');
var configuration = require('./strategies/restore/configuration.js');
var links = require('./strategies/restore/links.js');
function _interopNamespaceDefault(e) {
var n = Object.create(null);
if (e) {
Object.keys(e).forEach(function (k) {
if (k !== 'default') {
var d = Object.getOwnPropertyDescriptor(e, k);
Object.defineProperty(n, k, d.get ? d : {
enumerable: true,
get: function () { return e[k]; }
});
}
});
}
n.default = e;
return Object.freeze(n);
}
var fse__namespace = /*#__PURE__*/_interopNamespaceDefault(fse);
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
const VALID_CONFLICT_STRATEGIES = [
'restore'
];
const DEFAULT_CONFLICT_STRATEGY = 'restore';
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), /**
* The entities mapper is used to map old entities to their new IDs
*/ _entitiesMapper = /*#__PURE__*/ _class_private_field_loose_key("_entitiesMapper"), // TODO: either move this to restore strategy, or restore strategy should given access to these instead of repeating the logic possibly in a different way
_areAssetsIncluded = /*#__PURE__*/ _class_private_field_loose_key("_areAssetsIncluded"), _isContentTypeIncluded = /*#__PURE__*/ _class_private_field_loose_key("_isContentTypeIncluded"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), _validateOptions = /*#__PURE__*/ _class_private_field_loose_key("_validateOptions"), _deleteFromRestoreOptions = /*#__PURE__*/ _class_private_field_loose_key("_deleteFromRestoreOptions"), _deleteAllAssets = /*#__PURE__*/ _class_private_field_loose_key("_deleteAllAssets"), _handleAssetsBackup = /*#__PURE__*/ _class_private_field_loose_key("_handleAssetsBackup"), _removeAssetsBackup = /*#__PURE__*/ _class_private_field_loose_key("_removeAssetsBackup");
class LocalStrapiDestinationProvider {
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
_class_private_field_loose_base(this, _validateOptions)[_validateOptions]();
this.strapi = await this.options.getStrapi();
if (!this.strapi) {
throw new providers.ProviderInitializationError('Could not access local strapi');
}
this.strapi.db.lifecycles.disable();
this.transaction = transaction.createTransaction(this.strapi);
}
async close() {
const { autoDestroy } = this.options;
providers$1.assertValidStrapi(this.strapi);
this.transaction?.end();
this.strapi.db.lifecycles.enable();
// Basically `!== false` but more deterministic
if (autoDestroy === undefined || autoDestroy === true) {
await this.strapi?.destroy();
}
}
async rollback() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('Rolling back transaction');
await this.transaction?.rollback();
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('Rolled back transaction');
}
async beforeTransfer() {
if (!this.strapi) {
throw new Error('Strapi instance not found');
}
await this.transaction?.attach(async (trx)=>{
try {
if (this.options.strategy === 'restore') {
await _class_private_field_loose_base(this, _handleAssetsBackup)[_handleAssetsBackup]();
await _class_private_field_loose_base(this, _deleteAllAssets)[_deleteAllAssets](trx);
await _class_private_field_loose_base(this, _deleteFromRestoreOptions)[_deleteFromRestoreOptions]();
}
} catch (error) {
throw new Error(`restore failed ${error}`);
}
});
}
getMetadata() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting metadata');
providers$1.assertValidStrapi(this.strapi, 'Not able to get Schemas');
const strapiVersion = this.strapi.config.get('info.strapi');
const createdAt = new Date().toISOString();
return {
createdAt,
strapi: {
version: strapiVersion
}
};
}
getSchemas() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting schema');
providers$1.assertValidStrapi(this.strapi, 'Not able to get Schemas');
const schemas = schema.schemasToValidJSON({
...this.strapi.contentTypes,
...this.strapi.components
});
return schema.mapSchemasValues(schemas);
}
createEntitiesWriteStream() {
providers$1.assertValidStrapi(this.strapi, 'Not able to import entities');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating entities stream');
const { strategy } = this.options;
const updateMappingTable = (type, oldID, newID)=>{
if (!_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper][type]) {
_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper][type] = {};
}
Object.assign(_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper][type], {
[oldID]: newID
});
};
if (strategy === 'restore') {
return entities.createEntitiesWriteStream({
strapi: this.strapi,
updateMappingTable,
transaction: this.transaction
});
}
throw new providers.ProviderValidationError(`Invalid strategy ${this.options.strategy}`, {
check: 'strategy',
strategy: this.options.strategy,
validStrategies: VALID_CONFLICT_STRATEGIES
});
}
// TODO: Move this logic to the restore strategy
async createAssetsWriteStream() {
providers$1.assertValidStrapi(this.strapi, 'Not able to stream Assets');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets write stream');
if (!_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded]()) {
throw new providers.ProviderTransferError('Attempting to transfer assets when `assets` is not set in restore options');
}
const removeAssetsBackup = _class_private_field_loose_base(this, _removeAssetsBackup)[_removeAssetsBackup].bind(this);
const strapi = this.strapi;
const transaction = this.transaction;
const fileEntitiesMapper = _class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper]['plugin::upload.file'];
const restoreMediaEntitiesContent = _class_private_field_loose_base(this, _isContentTypeIncluded)[_isContentTypeIncluded]('plugin::upload.file');
return new stream.Writable({
objectMode: true,
async final (next) {
// Delete the backup folder
await removeAssetsBackup();
next();
},
async write (chunk, _encoding, callback) {
await transaction?.attach(async ()=>{
const uploadData = {
...chunk.metadata,
stream: stream.Readable.from(chunk.stream),
buffer: chunk?.buffer
};
const provider = strapi.config.get('plugin::upload').provider;
const fileId = fileEntitiesMapper?.[uploadData.id];
if (!fileId) {
return callback(new Error(`File ID not found for ID: ${uploadData.id}`));
}
try {
await strapi.plugin('upload').provider.uploadStream(uploadData);
// if we're not supposed to transfer the associated entities, stop here
if (!restoreMediaEntitiesContent) {
return callback();
}
// Files formats are stored within the parent file entity
if (uploadData?.type) {
const entry = await strapi.db.query('plugin::upload.file').findOne({
where: {
id: fileId
}
});
if (!entry) {
throw new Error('file not found');
}
const specificFormat = entry?.formats?.[uploadData.type];
if (specificFormat) {
specificFormat.url = uploadData.url;
}
await strapi.db.query('plugin::upload.file').update({
where: {
id: entry.id
},
data: {
formats: entry.formats,
provider
}
});
return callback();
}
const entry = await strapi.db.query('plugin::upload.file').findOne({
where: {
id: fileId
}
});
if (!entry) {
throw new Error('file not found');
}
entry.url = uploadData.url;
await strapi.db.query('plugin::upload.file').update({
where: {
id: entry.id
},
data: {
url: entry.url,
provider
}
});
return callback();
} catch (error) {
return callback(new Error(`Error while uploading asset ${chunk.filename} ${error}`));
}
});
}
});
}
async createConfigurationWriteStream() {
providers$1.assertValidStrapi(this.strapi, 'Not able to stream Configurations');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating configuration write stream');
const { strategy } = this.options;
if (strategy === 'restore') {
return configuration.createConfigurationWriteStream(this.strapi, this.transaction);
}
throw new providers.ProviderValidationError(`Invalid strategy ${strategy}`, {
check: 'strategy',
strategy,
validStrategies: VALID_CONFLICT_STRATEGIES
});
}
async createLinksWriteStream() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating links write stream');
if (!this.strapi) {
throw new Error('Not able to stream links. Strapi instance not found');
}
const { strategy } = this.options;
const mapID = (uid, id)=>_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper][uid]?.[id];
if (strategy === 'restore') {
return links.createLinksWriteStream(mapID, this.strapi, this.transaction, this.onWarning);
}
throw new providers.ProviderValidationError(`Invalid strategy ${strategy}`, {
check: 'strategy',
strategy,
validStrategies: VALID_CONFLICT_STRATEGIES
});
}
constructor(options){
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _validateOptions, {
value: validateOptions
});
Object.defineProperty(this, _deleteFromRestoreOptions, {
value: deleteFromRestoreOptions
});
Object.defineProperty(this, _deleteAllAssets, {
value: deleteAllAssets
});
Object.defineProperty(this, _handleAssetsBackup, {
value: handleAssetsBackup
});
Object.defineProperty(this, _removeAssetsBackup, {
value: removeAssetsBackup
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
Object.defineProperty(this, _entitiesMapper, {
writable: true,
value: void 0
});
Object.defineProperty(this, _areAssetsIncluded, {
writable: true,
value: void 0
});
Object.defineProperty(this, _isContentTypeIncluded, {
writable: true,
value: void 0
});
this.name = 'destination::local-strapi';
this.type = 'destination';
_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded] = ()=>{
return this.options.restore?.assets;
};
_class_private_field_loose_base(this, _isContentTypeIncluded)[_isContentTypeIncluded] = (type)=>{
const notIncluded = this.options.restore?.entities?.include && !this.options.restore?.entities?.include?.includes(type);
const excluded = this.options.restore?.entities?.exclude && this.options.restore?.entities.exclude.includes(type);
return !excluded && !notIncluded;
};
this.options = options;
_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper] = {};
this.uploadsBackupDirectoryName = `uploads_backup_${Date.now()}`;
}
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'local-destination-provider'
},
kind: 'info'
});
}
function validateOptions() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('validating options');
if (!VALID_CONFLICT_STRATEGIES.includes(this.options.strategy)) {
throw new providers.ProviderValidationError(`Invalid strategy ${this.options.strategy}`, {
check: 'strategy',
strategy: this.options.strategy,
validStrategies: VALID_CONFLICT_STRATEGIES
});
}
// require restore options when using restore
if (this.options.strategy === 'restore' && !this.options.restore) {
throw new providers.ProviderValidationError('Missing restore options');
}
}
async function deleteFromRestoreOptions() {
providers$1.assertValidStrapi(this.strapi);
if (!this.options.restore) {
throw new providers.ProviderValidationError('Missing restore options');
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('deleting record ');
return index.deleteRecords(this.strapi, this.options.restore);
}
async function deleteAllAssets(trx) {
providers$1.assertValidStrapi(this.strapi);
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('deleting all assets');
// if we're not restoring files, don't touch the files
if (!_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded]()) {
return;
}
const stream = this.strapi.db// Create a query builder instance (default type is 'select')
.queryBuilder('plugin::upload.file')// Fetch all columns
.select('*')// Attach the transaction
.transacting(trx)// Get a readable stream
.stream();
// TODO use bulk delete when exists in providers
for await (const file of stream){
await this.strapi.plugin('upload').provider.delete(file);
if (file.formats) {
for (const fileFormat of Object.values(file.formats)){
await this.strapi.plugin('upload').provider.delete(fileFormat);
}
}
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('deleted all assets');
}
async function handleAssetsBackup() {
providers$1.assertValidStrapi(this.strapi, 'Not able to create the assets backup');
// if we're not restoring assets, don't back them up because they won't be touched
if (!_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded]()) {
return;
}
if (this.strapi.config.get('plugin::upload').provider === 'local') {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets backup directory');
const assetsDirectory = path.join(this.strapi.dirs.static.public, 'uploads');
const backupDirectory = path.join(this.strapi.dirs.static.public, this.uploadsBackupDirectoryName);
try {
// Check access before attempting to do anything
await fse__namespace.access(assetsDirectory, // eslint-disable-next-line no-bitwise
fse__namespace.constants.W_OK | fse__namespace.constants.R_OK | fse__namespace.constants.F_OK);
// eslint-disable-next-line no-bitwise
await fse__namespace.access(path.join(assetsDirectory, '..'), fse__namespace.constants.W_OK | fse__namespace.constants.R_OK);
await fse__namespace.move(assetsDirectory, backupDirectory);
await fse__namespace.mkdir(assetsDirectory);
// Create a .gitkeep file to ensure the directory is not empty
await fse__namespace.outputFile(path.join(assetsDirectory, '.gitkeep'), '');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`created assets backup directory ${backupDirectory}`);
} catch (err) {
throw new providers.ProviderTransferError('The backup folder for the assets could not be created inside the public folder. Please ensure Strapi has write permissions on the public directory', {
code: 'ASSETS_DIRECTORY_ERR'
});
}
return backupDirectory;
}
}
async function removeAssetsBackup() {
providers$1.assertValidStrapi(this.strapi, 'Not able to remove Assets');
// if we're not restoring assets, don't back them up because they won't be touched
if (!_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded]()) {
return;
}
// TODO: this should catch all thrown errors and bubble it up to engine so it can be reported as a non-fatal diagnostic message telling the user they may need to manually delete assets
if (this.strapi.config.get('plugin::upload').provider === 'local') {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('removing assets backup');
providers$1.assertValidStrapi(this.strapi);
const backupDirectory = path.join(this.strapi.dirs.static.public, this.uploadsBackupDirectoryName);
await fse__namespace.rm(backupDirectory, {
recursive: true,
force: true
});
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('successfully removed assets backup');
}
}
const createLocalStrapiDestinationProvider = (options)=>{
return new LocalStrapiDestinationProvider(options);
};
exports.DEFAULT_CONFLICT_STRATEGY = DEFAULT_CONFLICT_STRATEGY;
exports.VALID_CONFLICT_STRATEGIES = VALID_CONFLICT_STRATEGIES;
exports.createLocalStrapiDestinationProvider = createLocalStrapiDestinationProvider;
//# sourceMappingURL=index.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,396 @@
import { Writable, Readable } from 'stream';
import path from 'path';
import * as fse from 'fs-extra';
import { deleteRecords } from './strategies/restore/index.mjs';
import 'crypto';
import 'lodash/fp';
import { schemasToValidJSON, mapSchemasValues } from '../../../utils/schema.mjs';
import { createTransaction } from '../../../utils/transaction.mjs';
import 'events';
import { ProviderInitializationError, ProviderValidationError, ProviderTransferError } from '../../../errors/providers.mjs';
import { assertValidStrapi } from '../../../utils/providers.mjs';
import { createEntitiesWriteStream } from './strategies/restore/entities.mjs';
import { createConfigurationWriteStream } from './strategies/restore/configuration.mjs';
import { createLinksWriteStream } from './strategies/restore/links.mjs';
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
const VALID_CONFLICT_STRATEGIES = [
'restore'
];
const DEFAULT_CONFLICT_STRATEGY = 'restore';
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), /**
* The entities mapper is used to map old entities to their new IDs
*/ _entitiesMapper = /*#__PURE__*/ _class_private_field_loose_key("_entitiesMapper"), // TODO: either move this to restore strategy, or restore strategy should given access to these instead of repeating the logic possibly in a different way
_areAssetsIncluded = /*#__PURE__*/ _class_private_field_loose_key("_areAssetsIncluded"), _isContentTypeIncluded = /*#__PURE__*/ _class_private_field_loose_key("_isContentTypeIncluded"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), _validateOptions = /*#__PURE__*/ _class_private_field_loose_key("_validateOptions"), _deleteFromRestoreOptions = /*#__PURE__*/ _class_private_field_loose_key("_deleteFromRestoreOptions"), _deleteAllAssets = /*#__PURE__*/ _class_private_field_loose_key("_deleteAllAssets"), _handleAssetsBackup = /*#__PURE__*/ _class_private_field_loose_key("_handleAssetsBackup"), _removeAssetsBackup = /*#__PURE__*/ _class_private_field_loose_key("_removeAssetsBackup");
class LocalStrapiDestinationProvider {
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
_class_private_field_loose_base(this, _validateOptions)[_validateOptions]();
this.strapi = await this.options.getStrapi();
if (!this.strapi) {
throw new ProviderInitializationError('Could not access local strapi');
}
this.strapi.db.lifecycles.disable();
this.transaction = createTransaction(this.strapi);
}
async close() {
const { autoDestroy } = this.options;
assertValidStrapi(this.strapi);
this.transaction?.end();
this.strapi.db.lifecycles.enable();
// Basically `!== false` but more deterministic
if (autoDestroy === undefined || autoDestroy === true) {
await this.strapi?.destroy();
}
}
async rollback() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('Rolling back transaction');
await this.transaction?.rollback();
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('Rolled back transaction');
}
async beforeTransfer() {
if (!this.strapi) {
throw new Error('Strapi instance not found');
}
await this.transaction?.attach(async (trx)=>{
try {
if (this.options.strategy === 'restore') {
await _class_private_field_loose_base(this, _handleAssetsBackup)[_handleAssetsBackup]();
await _class_private_field_loose_base(this, _deleteAllAssets)[_deleteAllAssets](trx);
await _class_private_field_loose_base(this, _deleteFromRestoreOptions)[_deleteFromRestoreOptions]();
}
} catch (error) {
throw new Error(`restore failed ${error}`);
}
});
}
getMetadata() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting metadata');
assertValidStrapi(this.strapi, 'Not able to get Schemas');
const strapiVersion = this.strapi.config.get('info.strapi');
const createdAt = new Date().toISOString();
return {
createdAt,
strapi: {
version: strapiVersion
}
};
}
getSchemas() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting schema');
assertValidStrapi(this.strapi, 'Not able to get Schemas');
const schemas = schemasToValidJSON({
...this.strapi.contentTypes,
...this.strapi.components
});
return mapSchemasValues(schemas);
}
createEntitiesWriteStream() {
assertValidStrapi(this.strapi, 'Not able to import entities');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating entities stream');
const { strategy } = this.options;
const updateMappingTable = (type, oldID, newID)=>{
if (!_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper][type]) {
_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper][type] = {};
}
Object.assign(_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper][type], {
[oldID]: newID
});
};
if (strategy === 'restore') {
return createEntitiesWriteStream({
strapi: this.strapi,
updateMappingTable,
transaction: this.transaction
});
}
throw new ProviderValidationError(`Invalid strategy ${this.options.strategy}`, {
check: 'strategy',
strategy: this.options.strategy,
validStrategies: VALID_CONFLICT_STRATEGIES
});
}
// TODO: Move this logic to the restore strategy
async createAssetsWriteStream() {
assertValidStrapi(this.strapi, 'Not able to stream Assets');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets write stream');
if (!_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded]()) {
throw new ProviderTransferError('Attempting to transfer assets when `assets` is not set in restore options');
}
const removeAssetsBackup = _class_private_field_loose_base(this, _removeAssetsBackup)[_removeAssetsBackup].bind(this);
const strapi = this.strapi;
const transaction = this.transaction;
const fileEntitiesMapper = _class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper]['plugin::upload.file'];
const restoreMediaEntitiesContent = _class_private_field_loose_base(this, _isContentTypeIncluded)[_isContentTypeIncluded]('plugin::upload.file');
return new Writable({
objectMode: true,
async final (next) {
// Delete the backup folder
await removeAssetsBackup();
next();
},
async write (chunk, _encoding, callback) {
await transaction?.attach(async ()=>{
const uploadData = {
...chunk.metadata,
stream: Readable.from(chunk.stream),
buffer: chunk?.buffer
};
const provider = strapi.config.get('plugin::upload').provider;
const fileId = fileEntitiesMapper?.[uploadData.id];
if (!fileId) {
return callback(new Error(`File ID not found for ID: ${uploadData.id}`));
}
try {
await strapi.plugin('upload').provider.uploadStream(uploadData);
// if we're not supposed to transfer the associated entities, stop here
if (!restoreMediaEntitiesContent) {
return callback();
}
// Files formats are stored within the parent file entity
if (uploadData?.type) {
const entry = await strapi.db.query('plugin::upload.file').findOne({
where: {
id: fileId
}
});
if (!entry) {
throw new Error('file not found');
}
const specificFormat = entry?.formats?.[uploadData.type];
if (specificFormat) {
specificFormat.url = uploadData.url;
}
await strapi.db.query('plugin::upload.file').update({
where: {
id: entry.id
},
data: {
formats: entry.formats,
provider
}
});
return callback();
}
const entry = await strapi.db.query('plugin::upload.file').findOne({
where: {
id: fileId
}
});
if (!entry) {
throw new Error('file not found');
}
entry.url = uploadData.url;
await strapi.db.query('plugin::upload.file').update({
where: {
id: entry.id
},
data: {
url: entry.url,
provider
}
});
return callback();
} catch (error) {
return callback(new Error(`Error while uploading asset ${chunk.filename} ${error}`));
}
});
}
});
}
async createConfigurationWriteStream() {
assertValidStrapi(this.strapi, 'Not able to stream Configurations');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating configuration write stream');
const { strategy } = this.options;
if (strategy === 'restore') {
return createConfigurationWriteStream(this.strapi, this.transaction);
}
throw new ProviderValidationError(`Invalid strategy ${strategy}`, {
check: 'strategy',
strategy,
validStrategies: VALID_CONFLICT_STRATEGIES
});
}
async createLinksWriteStream() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating links write stream');
if (!this.strapi) {
throw new Error('Not able to stream links. Strapi instance not found');
}
const { strategy } = this.options;
const mapID = (uid, id)=>_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper][uid]?.[id];
if (strategy === 'restore') {
return createLinksWriteStream(mapID, this.strapi, this.transaction, this.onWarning);
}
throw new ProviderValidationError(`Invalid strategy ${strategy}`, {
check: 'strategy',
strategy,
validStrategies: VALID_CONFLICT_STRATEGIES
});
}
constructor(options){
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _validateOptions, {
value: validateOptions
});
Object.defineProperty(this, _deleteFromRestoreOptions, {
value: deleteFromRestoreOptions
});
Object.defineProperty(this, _deleteAllAssets, {
value: deleteAllAssets
});
Object.defineProperty(this, _handleAssetsBackup, {
value: handleAssetsBackup
});
Object.defineProperty(this, _removeAssetsBackup, {
value: removeAssetsBackup
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
Object.defineProperty(this, _entitiesMapper, {
writable: true,
value: void 0
});
Object.defineProperty(this, _areAssetsIncluded, {
writable: true,
value: void 0
});
Object.defineProperty(this, _isContentTypeIncluded, {
writable: true,
value: void 0
});
this.name = 'destination::local-strapi';
this.type = 'destination';
_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded] = ()=>{
return this.options.restore?.assets;
};
_class_private_field_loose_base(this, _isContentTypeIncluded)[_isContentTypeIncluded] = (type)=>{
const notIncluded = this.options.restore?.entities?.include && !this.options.restore?.entities?.include?.includes(type);
const excluded = this.options.restore?.entities?.exclude && this.options.restore?.entities.exclude.includes(type);
return !excluded && !notIncluded;
};
this.options = options;
_class_private_field_loose_base(this, _entitiesMapper)[_entitiesMapper] = {};
this.uploadsBackupDirectoryName = `uploads_backup_${Date.now()}`;
}
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'local-destination-provider'
},
kind: 'info'
});
}
function validateOptions() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('validating options');
if (!VALID_CONFLICT_STRATEGIES.includes(this.options.strategy)) {
throw new ProviderValidationError(`Invalid strategy ${this.options.strategy}`, {
check: 'strategy',
strategy: this.options.strategy,
validStrategies: VALID_CONFLICT_STRATEGIES
});
}
// require restore options when using restore
if (this.options.strategy === 'restore' && !this.options.restore) {
throw new ProviderValidationError('Missing restore options');
}
}
async function deleteFromRestoreOptions() {
assertValidStrapi(this.strapi);
if (!this.options.restore) {
throw new ProviderValidationError('Missing restore options');
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('deleting record ');
return deleteRecords(this.strapi, this.options.restore);
}
async function deleteAllAssets(trx) {
assertValidStrapi(this.strapi);
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('deleting all assets');
// if we're not restoring files, don't touch the files
if (!_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded]()) {
return;
}
const stream = this.strapi.db// Create a query builder instance (default type is 'select')
.queryBuilder('plugin::upload.file')// Fetch all columns
.select('*')// Attach the transaction
.transacting(trx)// Get a readable stream
.stream();
// TODO use bulk delete when exists in providers
for await (const file of stream){
await this.strapi.plugin('upload').provider.delete(file);
if (file.formats) {
for (const fileFormat of Object.values(file.formats)){
await this.strapi.plugin('upload').provider.delete(fileFormat);
}
}
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('deleted all assets');
}
async function handleAssetsBackup() {
assertValidStrapi(this.strapi, 'Not able to create the assets backup');
// if we're not restoring assets, don't back them up because they won't be touched
if (!_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded]()) {
return;
}
if (this.strapi.config.get('plugin::upload').provider === 'local') {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets backup directory');
const assetsDirectory = path.join(this.strapi.dirs.static.public, 'uploads');
const backupDirectory = path.join(this.strapi.dirs.static.public, this.uploadsBackupDirectoryName);
try {
// Check access before attempting to do anything
await fse.access(assetsDirectory, // eslint-disable-next-line no-bitwise
fse.constants.W_OK | fse.constants.R_OK | fse.constants.F_OK);
// eslint-disable-next-line no-bitwise
await fse.access(path.join(assetsDirectory, '..'), fse.constants.W_OK | fse.constants.R_OK);
await fse.move(assetsDirectory, backupDirectory);
await fse.mkdir(assetsDirectory);
// Create a .gitkeep file to ensure the directory is not empty
await fse.outputFile(path.join(assetsDirectory, '.gitkeep'), '');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`created assets backup directory ${backupDirectory}`);
} catch (err) {
throw new ProviderTransferError('The backup folder for the assets could not be created inside the public folder. Please ensure Strapi has write permissions on the public directory', {
code: 'ASSETS_DIRECTORY_ERR'
});
}
return backupDirectory;
}
}
async function removeAssetsBackup() {
assertValidStrapi(this.strapi, 'Not able to remove Assets');
// if we're not restoring assets, don't back them up because they won't be touched
if (!_class_private_field_loose_base(this, _areAssetsIncluded)[_areAssetsIncluded]()) {
return;
}
// TODO: this should catch all thrown errors and bubble it up to engine so it can be reported as a non-fatal diagnostic message telling the user they may need to manually delete assets
if (this.strapi.config.get('plugin::upload').provider === 'local') {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('removing assets backup');
assertValidStrapi(this.strapi);
const backupDirectory = path.join(this.strapi.dirs.static.public, this.uploadsBackupDirectoryName);
await fse.rm(backupDirectory, {
recursive: true,
force: true
});
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('successfully removed assets backup');
}
}
const createLocalStrapiDestinationProvider = (options)=>{
return new LocalStrapiDestinationProvider(options);
};
export { DEFAULT_CONFLICT_STRATEGY, VALID_CONFLICT_STRATEGIES, createLocalStrapiDestinationProvider };
//# sourceMappingURL=index.mjs.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,2 @@
export * as restore from './restore';
//# sourceMappingURL=index.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/strapi/providers/local-destination/strategies/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,WAAW,CAAC"}

View File

@@ -0,0 +1,7 @@
/// <reference types="node" />
import { Writable } from 'stream';
import type { Core } from '@strapi/types';
import { IConfiguration, Transaction } from '../../../../../../types';
export declare const restoreConfigs: (strapi: Core.Strapi, config: IConfiguration) => Promise<any>;
export declare const createConfigurationWriteStream: (strapi: Core.Strapi, transaction?: Transaction) => Promise<Writable>;
//# sourceMappingURL=configuration.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAGlC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAE1C,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAmBtE,eAAO,MAAM,cAAc,WAAkB,KAAK,MAAM,UAAU,cAAc,iBAQ/E,CAAC;AAEF,eAAO,MAAM,8BAA8B,WACjC,KAAK,MAAM,gBACL,WAAW,sBAyB1B,CAAC"}

View File

@@ -0,0 +1,52 @@
'use strict';
var stream = require('stream');
var fp = require('lodash/fp');
var chalk = require('chalk');
var providers = require('../../../../../errors/providers.js');
const omitInvalidCreationAttributes = fp.omit([
'id'
]);
const restoreCoreStore = async (strapi, values)=>{
const data = omitInvalidCreationAttributes(values);
return strapi.db.query('strapi::core-store').create({
data: {
...data,
value: JSON.stringify(data.value)
}
});
};
const restoreWebhooks = async (strapi, values)=>{
const data = omitInvalidCreationAttributes(values);
return strapi.db.query('strapi::webhook').create({
data
});
};
const restoreConfigs = async (strapi, config)=>{
if (config.type === 'core-store') {
return restoreCoreStore(strapi, config.value);
}
if (config.type === 'webhook') {
return restoreWebhooks(strapi, config.value);
}
};
const createConfigurationWriteStream = async (strapi, transaction)=>{
return new stream.Writable({
objectMode: true,
async write (config, _encoding, callback) {
await transaction?.attach(async ()=>{
try {
await restoreConfigs(strapi, config);
} catch (error) {
return callback(new providers.ProviderTransferError(`Failed to import ${chalk.yellowBright(config.type)} (${chalk.greenBright(config.value.id)}`));
}
callback();
});
}
});
};
exports.createConfigurationWriteStream = createConfigurationWriteStream;
exports.restoreConfigs = restoreConfigs;
//# sourceMappingURL=configuration.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"configuration.js","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/configuration.ts"],"sourcesContent":["import { Writable } from 'stream';\nimport { omit } from 'lodash/fp';\nimport chalk from 'chalk';\nimport type { Core } from '@strapi/types';\nimport { ProviderTransferError } from '../../../../../errors/providers';\nimport { IConfiguration, Transaction } from '../../../../../../types';\n\nconst omitInvalidCreationAttributes = omit(['id']);\n\nconst restoreCoreStore = async <T extends { value: unknown }>(strapi: Core.Strapi, values: T) => {\n const data = omitInvalidCreationAttributes(values);\n return strapi.db.query('strapi::core-store').create({\n data: {\n ...data,\n value: JSON.stringify(data.value),\n },\n });\n};\n\nconst restoreWebhooks = async <T extends { value: unknown }>(strapi: Core.Strapi, values: T) => {\n const data = omitInvalidCreationAttributes(values);\n return strapi.db.query('strapi::webhook').create({ data });\n};\n\nexport const restoreConfigs = async (strapi: Core.Strapi, config: IConfiguration) => {\n if (config.type === 'core-store') {\n return restoreCoreStore(strapi, config.value as { value: unknown });\n }\n\n if (config.type === 'webhook') {\n return restoreWebhooks(strapi, config.value as { value: unknown });\n }\n};\n\nexport const createConfigurationWriteStream = async (\n strapi: Core.Strapi,\n transaction?: Transaction\n) => {\n return new Writable({\n objectMode: true,\n async write<T extends { id: number }>(\n config: IConfiguration<T>,\n _encoding: BufferEncoding,\n callback: (error?: Error | null) => void\n ) {\n await transaction?.attach(async () => {\n try {\n await restoreConfigs(strapi, config);\n } catch (error) {\n return callback(\n new ProviderTransferError(\n `Failed to import ${chalk.yellowBright(config.type)} (${chalk.greenBright(\n config.value.id\n )}`\n )\n );\n }\n callback();\n });\n },\n });\n};\n"],"names":["omitInvalidCreationAttributes","omit","restoreCoreStore","strapi","values","data","db","query","create","value","JSON","stringify","restoreWebhooks","restoreConfigs","config","type","createConfigurationWriteStream","transaction","Writable","objectMode","write","_encoding","callback","attach","error","ProviderTransferError","chalk","yellowBright","greenBright","id"],"mappings":";;;;;;;AAOA,MAAMA,gCAAgCC,OAAK,CAAA;AAAC,IAAA;AAAK,CAAA,CAAA;AAEjD,MAAMC,gBAAAA,GAAmB,OAAqCC,MAAqBC,EAAAA,MAAAA,GAAAA;AACjF,IAAA,MAAMC,OAAOL,6BAA8BI,CAAAA,MAAAA,CAAAA;AAC3C,IAAA,OAAOD,OAAOG,EAAE,CAACC,KAAK,CAAC,oBAAA,CAAA,CAAsBC,MAAM,CAAC;QAClDH,IAAM,EAAA;AACJ,YAAA,GAAGA,IAAI;AACPI,YAAAA,KAAAA,EAAOC,IAAKC,CAAAA,SAAS,CAACN,IAAAA,CAAKI,KAAK;AAClC;AACF,KAAA,CAAA;AACF,CAAA;AAEA,MAAMG,eAAAA,GAAkB,OAAqCT,MAAqBC,EAAAA,MAAAA,GAAAA;AAChF,IAAA,MAAMC,OAAOL,6BAA8BI,CAAAA,MAAAA,CAAAA;AAC3C,IAAA,OAAOD,OAAOG,EAAE,CAACC,KAAK,CAAC,iBAAA,CAAA,CAAmBC,MAAM,CAAC;AAAEH,QAAAA;AAAK,KAAA,CAAA;AAC1D,CAAA;AAEO,MAAMQ,cAAiB,GAAA,OAAOV,MAAqBW,EAAAA,MAAAA,GAAAA;IACxD,IAAIA,MAAAA,CAAOC,IAAI,KAAK,YAAc,EAAA;QAChC,OAAOb,gBAAAA,CAAiBC,MAAQW,EAAAA,MAAAA,CAAOL,KAAK,CAAA;AAC9C;IAEA,IAAIK,MAAAA,CAAOC,IAAI,KAAK,SAAW,EAAA;QAC7B,OAAOH,eAAAA,CAAgBT,MAAQW,EAAAA,MAAAA,CAAOL,KAAK,CAAA;AAC7C;AACF;AAEO,MAAMO,8BAAiC,GAAA,OAC5Cb,MACAc,EAAAA,WAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,eAAS,CAAA;QAClBC,UAAY,EAAA,IAAA;AACZ,QAAA,MAAMC,KACJN,CAAAA,CAAAA,MAAyB,EACzBO,SAAyB,EACzBC,QAAwC,EAAA;AAExC,YAAA,MAAML,aAAaM,MAAO,CAAA,UAAA;gBACxB,IAAI;AACF,oBAAA,MAAMV,eAAeV,MAAQW,EAAAA,MAAAA,CAAAA;AAC/B,iBAAA,CAAE,OAAOU,KAAO,EAAA;oBACd,OAAOF,QAAAA,CACL,IAAIG,+BACF,CAAA,CAAC,iBAAiB,EAAEC,KAAAA,CAAMC,YAAY,CAACb,MAAOC,CAAAA,IAAI,EAAE,EAAE,EAAEW,MAAME,WAAW,CACvEd,OAAOL,KAAK,CAACoB,EAAE,CAAA,CACf,CAAC,CAAA,CAAA;AAGT;AACAP,gBAAAA,QAAAA,EAAAA;AACF,aAAA,CAAA;AACF;AACF,KAAA,CAAA;AACF;;;;;"}

View File

@@ -0,0 +1,49 @@
import { Writable } from 'stream';
import { omit } from 'lodash/fp';
import chalk from 'chalk';
import { ProviderTransferError } from '../../../../../errors/providers.mjs';
const omitInvalidCreationAttributes = omit([
'id'
]);
const restoreCoreStore = async (strapi, values)=>{
const data = omitInvalidCreationAttributes(values);
return strapi.db.query('strapi::core-store').create({
data: {
...data,
value: JSON.stringify(data.value)
}
});
};
const restoreWebhooks = async (strapi, values)=>{
const data = omitInvalidCreationAttributes(values);
return strapi.db.query('strapi::webhook').create({
data
});
};
const restoreConfigs = async (strapi, config)=>{
if (config.type === 'core-store') {
return restoreCoreStore(strapi, config.value);
}
if (config.type === 'webhook') {
return restoreWebhooks(strapi, config.value);
}
};
const createConfigurationWriteStream = async (strapi, transaction)=>{
return new Writable({
objectMode: true,
async write (config, _encoding, callback) {
await transaction?.attach(async ()=>{
try {
await restoreConfigs(strapi, config);
} catch (error) {
return callback(new ProviderTransferError(`Failed to import ${chalk.yellowBright(config.type)} (${chalk.greenBright(config.value.id)}`));
}
callback();
});
}
});
};
export { createConfigurationWriteStream, restoreConfigs };
//# sourceMappingURL=configuration.mjs.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"configuration.mjs","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/configuration.ts"],"sourcesContent":["import { Writable } from 'stream';\nimport { omit } from 'lodash/fp';\nimport chalk from 'chalk';\nimport type { Core } from '@strapi/types';\nimport { ProviderTransferError } from '../../../../../errors/providers';\nimport { IConfiguration, Transaction } from '../../../../../../types';\n\nconst omitInvalidCreationAttributes = omit(['id']);\n\nconst restoreCoreStore = async <T extends { value: unknown }>(strapi: Core.Strapi, values: T) => {\n const data = omitInvalidCreationAttributes(values);\n return strapi.db.query('strapi::core-store').create({\n data: {\n ...data,\n value: JSON.stringify(data.value),\n },\n });\n};\n\nconst restoreWebhooks = async <T extends { value: unknown }>(strapi: Core.Strapi, values: T) => {\n const data = omitInvalidCreationAttributes(values);\n return strapi.db.query('strapi::webhook').create({ data });\n};\n\nexport const restoreConfigs = async (strapi: Core.Strapi, config: IConfiguration) => {\n if (config.type === 'core-store') {\n return restoreCoreStore(strapi, config.value as { value: unknown });\n }\n\n if (config.type === 'webhook') {\n return restoreWebhooks(strapi, config.value as { value: unknown });\n }\n};\n\nexport const createConfigurationWriteStream = async (\n strapi: Core.Strapi,\n transaction?: Transaction\n) => {\n return new Writable({\n objectMode: true,\n async write<T extends { id: number }>(\n config: IConfiguration<T>,\n _encoding: BufferEncoding,\n callback: (error?: Error | null) => void\n ) {\n await transaction?.attach(async () => {\n try {\n await restoreConfigs(strapi, config);\n } catch (error) {\n return callback(\n new ProviderTransferError(\n `Failed to import ${chalk.yellowBright(config.type)} (${chalk.greenBright(\n config.value.id\n )}`\n )\n );\n }\n callback();\n });\n },\n });\n};\n"],"names":["omitInvalidCreationAttributes","omit","restoreCoreStore","strapi","values","data","db","query","create","value","JSON","stringify","restoreWebhooks","restoreConfigs","config","type","createConfigurationWriteStream","transaction","Writable","objectMode","write","_encoding","callback","attach","error","ProviderTransferError","chalk","yellowBright","greenBright","id"],"mappings":";;;;;AAOA,MAAMA,gCAAgCC,IAAK,CAAA;AAAC,IAAA;AAAK,CAAA,CAAA;AAEjD,MAAMC,gBAAAA,GAAmB,OAAqCC,MAAqBC,EAAAA,MAAAA,GAAAA;AACjF,IAAA,MAAMC,OAAOL,6BAA8BI,CAAAA,MAAAA,CAAAA;AAC3C,IAAA,OAAOD,OAAOG,EAAE,CAACC,KAAK,CAAC,oBAAA,CAAA,CAAsBC,MAAM,CAAC;QAClDH,IAAM,EAAA;AACJ,YAAA,GAAGA,IAAI;AACPI,YAAAA,KAAAA,EAAOC,IAAKC,CAAAA,SAAS,CAACN,IAAAA,CAAKI,KAAK;AAClC;AACF,KAAA,CAAA;AACF,CAAA;AAEA,MAAMG,eAAAA,GAAkB,OAAqCT,MAAqBC,EAAAA,MAAAA,GAAAA;AAChF,IAAA,MAAMC,OAAOL,6BAA8BI,CAAAA,MAAAA,CAAAA;AAC3C,IAAA,OAAOD,OAAOG,EAAE,CAACC,KAAK,CAAC,iBAAA,CAAA,CAAmBC,MAAM,CAAC;AAAEH,QAAAA;AAAK,KAAA,CAAA;AAC1D,CAAA;AAEO,MAAMQ,cAAiB,GAAA,OAAOV,MAAqBW,EAAAA,MAAAA,GAAAA;IACxD,IAAIA,MAAAA,CAAOC,IAAI,KAAK,YAAc,EAAA;QAChC,OAAOb,gBAAAA,CAAiBC,MAAQW,EAAAA,MAAAA,CAAOL,KAAK,CAAA;AAC9C;IAEA,IAAIK,MAAAA,CAAOC,IAAI,KAAK,SAAW,EAAA;QAC7B,OAAOH,eAAAA,CAAgBT,MAAQW,EAAAA,MAAAA,CAAOL,KAAK,CAAA;AAC7C;AACF;AAEO,MAAMO,8BAAiC,GAAA,OAC5Cb,MACAc,EAAAA,WAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,QAAS,CAAA;QAClBC,UAAY,EAAA,IAAA;AACZ,QAAA,MAAMC,KACJN,CAAAA,CAAAA,MAAyB,EACzBO,SAAyB,EACzBC,QAAwC,EAAA;AAExC,YAAA,MAAML,aAAaM,MAAO,CAAA,UAAA;gBACxB,IAAI;AACF,oBAAA,MAAMV,eAAeV,MAAQW,EAAAA,MAAAA,CAAAA;AAC/B,iBAAA,CAAE,OAAOU,KAAO,EAAA;oBACd,OAAOF,QAAAA,CACL,IAAIG,qBACF,CAAA,CAAC,iBAAiB,EAAEC,KAAAA,CAAMC,YAAY,CAACb,MAAOC,CAAAA,IAAI,EAAE,EAAE,EAAEW,MAAME,WAAW,CACvEd,OAAOL,KAAK,CAACoB,EAAE,CAAA,CACf,CAAC,CAAA,CAAA;AAGT;AACAP,gBAAAA,QAAAA,EAAAA;AACF,aAAA,CAAA;AACF;AACF,KAAA,CAAA;AACF;;;;"}

View File

@@ -0,0 +1,12 @@
/// <reference types="node" />
import { Writable } from 'stream';
import type { Core, UID } from '@strapi/types';
import type { Transaction } from '../../../../../../types';
interface IEntitiesRestoreStreamOptions {
strapi: Core.Strapi;
updateMappingTable<TSchemaUID extends UID.Schema>(type: TSchemaUID, oldID: number, newID: number): void;
transaction?: Transaction;
}
export declare const createEntitiesWriteStream: (options: IEntitiesRestoreStreamOptions) => Writable;
export {};
//# sourceMappingURL=entities.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"entities.d.ts","sourceRoot":"","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/entities.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,eAAe,CAAC;AAK/C,OAAO,KAAK,EAAW,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAKpE,UAAU,6BAA6B;IACrC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC;IACpB,kBAAkB,CAAC,UAAU,SAAS,GAAG,CAAC,MAAM,EAC9C,IAAI,EAAE,UAAU,EAChB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,IAAI,CAAC;IACR,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED,eAAO,MAAM,yBAAyB,YAAa,6BAA6B,aAqD/E,CAAC"}

View File

@@ -0,0 +1,64 @@
'use strict';
var stream = require('stream');
var fp = require('lodash/fp');
var providers = require('../../../../../errors/providers.js');
require('crypto');
var json = require('../../../../../utils/json.js');
require('events');
var entity = require('../../../../queries/entity.js');
var components = require('../../../../../utils/components.js');
const createEntitiesWriteStream = (options)=>{
const { strapi, updateMappingTable, transaction } = options;
const query = entity.createEntityQuery(strapi);
return new stream.Writable({
objectMode: true,
async write (entity, _encoding, callback) {
await transaction?.attach(async ()=>{
const { type, id, data } = entity;
const { create, getDeepPopulateComponentLikeQuery } = query(type);
const contentType = strapi.getModel(type);
try {
const created = await create({
data,
populate: getDeepPopulateComponentLikeQuery(contentType, {
select: 'id'
}),
select: 'id'
});
// Compute differences between original & new entities
const diffs = json.diff(data, created);
updateMappingTable(type, id, created.id);
// For each difference found on an ID attribute,
// update the mapping the table accordingly
diffs.forEach((diff)=>{
if (diff.kind === 'modified' && fp.last(diff.path) === 'id' && 'kind' in contentType) {
const target = components.resolveComponentUID({
paths: diff.path,
data,
contentType,
strapi
});
// If no type is found for the given path, then ignore the diff
if (!target) {
return;
}
const [oldID, newID] = diff.values;
updateMappingTable(target, oldID, newID);
}
});
} catch (e) {
if (e instanceof Error) {
return callback(e);
}
return callback(new providers.ProviderTransferError(`Failed to create "${type}" (${id})`));
}
return callback(null);
});
}
});
};
exports.createEntitiesWriteStream = createEntitiesWriteStream;
//# sourceMappingURL=entities.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"entities.js","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/entities.ts"],"sourcesContent":["import { Writable } from 'stream';\nimport type { Core, UID } from '@strapi/types';\n\nimport { last } from 'lodash/fp';\n\nimport { ProviderTransferError } from '../../../../../errors/providers';\nimport type { IEntity, Transaction } from '../../../../../../types';\nimport { json } from '../../../../../utils';\nimport * as queries from '../../../../queries';\nimport { resolveComponentUID } from '../../../../../utils/components';\n\ninterface IEntitiesRestoreStreamOptions {\n strapi: Core.Strapi;\n updateMappingTable<TSchemaUID extends UID.Schema>(\n type: TSchemaUID,\n oldID: number,\n newID: number\n ): void;\n transaction?: Transaction;\n}\n\nexport const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {\n const { strapi, updateMappingTable, transaction } = options;\n const query = queries.entity.createEntityQuery(strapi);\n\n return new Writable({\n objectMode: true,\n\n async write(entity: IEntity, _encoding, callback) {\n await transaction?.attach(async () => {\n const { type, id, data } = entity;\n const { create, getDeepPopulateComponentLikeQuery } = query(type);\n const contentType = strapi.getModel(type);\n\n try {\n const created = await create({\n data,\n populate: getDeepPopulateComponentLikeQuery(contentType, { select: 'id' }),\n select: 'id',\n });\n\n // Compute differences between original & new entities\n const diffs = json.diff(data, created);\n\n updateMappingTable(type, id, created.id);\n\n // For each difference found on an ID attribute,\n // update the mapping the table accordingly\n diffs.forEach((diff) => {\n if (diff.kind === 'modified' && last(diff.path) === 'id' && 'kind' in contentType) {\n const target = resolveComponentUID({ paths: diff.path, data, contentType, strapi });\n\n // If no type is found for the given path, then ignore the diff\n if (!target) {\n return;\n }\n\n const [oldID, newID] = diff.values as [number, number];\n\n updateMappingTable(target, oldID, newID);\n }\n });\n } catch (e) {\n if (e instanceof Error) {\n return callback(e);\n }\n\n return callback(new ProviderTransferError(`Failed to create \"${type}\" (${id})`));\n }\n\n return callback(null);\n });\n },\n });\n};\n"],"names":["createEntitiesWriteStream","options","strapi","updateMappingTable","transaction","query","queries","Writable","objectMode","write","entity","_encoding","callback","attach","type","id","data","create","getDeepPopulateComponentLikeQuery","contentType","getModel","created","populate","select","diffs","json","forEach","diff","kind","last","path","target","resolveComponentUID","paths","oldID","newID","values","e","Error","ProviderTransferError"],"mappings":";;;;;;;;;;;AAqBO,MAAMA,4BAA4B,CAACC,OAAAA,GAAAA;AACxC,IAAA,MAAM,EAAEC,MAAM,EAAEC,kBAAkB,EAAEC,WAAW,EAAE,GAAGH,OAAAA;AACpD,IAAA,MAAMI,KAAQC,GAAAA,wBAAgC,CAACJ,MAAAA,CAAAA;AAE/C,IAAA,OAAO,IAAIK,eAAS,CAAA;QAClBC,UAAY,EAAA,IAAA;AAEZ,QAAA,MAAMC,KAAMC,CAAAA,CAAAA,MAAe,EAAEC,SAAS,EAAEC,QAAQ,EAAA;AAC9C,YAAA,MAAMR,aAAaS,MAAO,CAAA,UAAA;AACxB,gBAAA,MAAM,EAAEC,IAAI,EAAEC,EAAE,EAAEC,IAAI,EAAE,GAAGN,MAAAA;AAC3B,gBAAA,MAAM,EAAEO,MAAM,EAAEC,iCAAiC,EAAE,GAAGb,KAAMS,CAAAA,IAAAA,CAAAA;gBAC5D,MAAMK,WAAAA,GAAcjB,MAAOkB,CAAAA,QAAQ,CAACN,IAAAA,CAAAA;gBAEpC,IAAI;oBACF,MAAMO,OAAAA,GAAU,MAAMJ,MAAO,CAAA;AAC3BD,wBAAAA,IAAAA;AACAM,wBAAAA,QAAAA,EAAUJ,kCAAkCC,WAAa,EAAA;4BAAEI,MAAQ,EAAA;AAAK,yBAAA,CAAA;wBACxEA,MAAQ,EAAA;AACV,qBAAA,CAAA;;AAGA,oBAAA,MAAMC,KAAQC,GAAAA,SAAS,CAACT,IAAMK,EAAAA,OAAAA,CAAAA;oBAE9BlB,kBAAmBW,CAAAA,IAAAA,EAAMC,EAAIM,EAAAA,OAAAA,CAAQN,EAAE,CAAA;;;oBAIvCS,KAAME,CAAAA,OAAO,CAAC,CAACC,IAAAA,GAAAA;wBACb,IAAIA,IAAAA,CAAKC,IAAI,KAAK,UAAcC,IAAAA,OAAAA,CAAKF,KAAKG,IAAI,CAAA,KAAM,IAAQ,IAAA,MAAA,IAAUX,WAAa,EAAA;AACjF,4BAAA,MAAMY,SAASC,8BAAoB,CAAA;AAAEC,gCAAAA,KAAAA,EAAON,KAAKG,IAAI;AAAEd,gCAAAA,IAAAA;AAAMG,gCAAAA,WAAAA;AAAajB,gCAAAA;AAAO,6BAAA,CAAA;;AAGjF,4BAAA,IAAI,CAAC6B,MAAQ,EAAA;AACX,gCAAA;AACF;AAEA,4BAAA,MAAM,CAACG,KAAAA,EAAOC,KAAM,CAAA,GAAGR,KAAKS,MAAM;AAElCjC,4BAAAA,kBAAAA,CAAmB4B,QAAQG,KAAOC,EAAAA,KAAAA,CAAAA;AACpC;AACF,qBAAA,CAAA;AACF,iBAAA,CAAE,OAAOE,CAAG,EAAA;AACV,oBAAA,IAAIA,aAAaC,KAAO,EAAA;AACtB,wBAAA,OAAO1B,QAASyB,CAAAA,CAAAA,CAAAA;AAClB;oBAEA,OAAOzB,QAAAA,CAAS,IAAI2B,+BAAAA,CAAsB,CAAC,kBAAkB,EAAEzB,IAAAA,CAAK,GAAG,EAAEC,EAAG,CAAA,CAAC,CAAC,CAAA,CAAA;AAChF;AAEA,gBAAA,OAAOH,QAAS,CAAA,IAAA,CAAA;AAClB,aAAA,CAAA;AACF;AACF,KAAA,CAAA;AACF;;;;"}

View File

@@ -0,0 +1,62 @@
import { Writable } from 'stream';
import { last } from 'lodash/fp';
import { ProviderTransferError } from '../../../../../errors/providers.mjs';
import 'crypto';
import { diff } from '../../../../../utils/json.mjs';
import 'events';
import { createEntityQuery } from '../../../../queries/entity.mjs';
import { resolveComponentUID } from '../../../../../utils/components.mjs';
const createEntitiesWriteStream = (options)=>{
const { strapi, updateMappingTable, transaction } = options;
const query = createEntityQuery(strapi);
return new Writable({
objectMode: true,
async write (entity, _encoding, callback) {
await transaction?.attach(async ()=>{
const { type, id, data } = entity;
const { create, getDeepPopulateComponentLikeQuery } = query(type);
const contentType = strapi.getModel(type);
try {
const created = await create({
data,
populate: getDeepPopulateComponentLikeQuery(contentType, {
select: 'id'
}),
select: 'id'
});
// Compute differences between original & new entities
const diffs = diff(data, created);
updateMappingTable(type, id, created.id);
// For each difference found on an ID attribute,
// update the mapping the table accordingly
diffs.forEach((diff)=>{
if (diff.kind === 'modified' && last(diff.path) === 'id' && 'kind' in contentType) {
const target = resolveComponentUID({
paths: diff.path,
data,
contentType,
strapi
});
// If no type is found for the given path, then ignore the diff
if (!target) {
return;
}
const [oldID, newID] = diff.values;
updateMappingTable(target, oldID, newID);
}
});
} catch (e) {
if (e instanceof Error) {
return callback(e);
}
return callback(new ProviderTransferError(`Failed to create "${type}" (${id})`));
}
return callback(null);
});
}
});
};
export { createEntitiesWriteStream };
//# sourceMappingURL=entities.mjs.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"entities.mjs","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/entities.ts"],"sourcesContent":["import { Writable } from 'stream';\nimport type { Core, UID } from '@strapi/types';\n\nimport { last } from 'lodash/fp';\n\nimport { ProviderTransferError } from '../../../../../errors/providers';\nimport type { IEntity, Transaction } from '../../../../../../types';\nimport { json } from '../../../../../utils';\nimport * as queries from '../../../../queries';\nimport { resolveComponentUID } from '../../../../../utils/components';\n\ninterface IEntitiesRestoreStreamOptions {\n strapi: Core.Strapi;\n updateMappingTable<TSchemaUID extends UID.Schema>(\n type: TSchemaUID,\n oldID: number,\n newID: number\n ): void;\n transaction?: Transaction;\n}\n\nexport const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {\n const { strapi, updateMappingTable, transaction } = options;\n const query = queries.entity.createEntityQuery(strapi);\n\n return new Writable({\n objectMode: true,\n\n async write(entity: IEntity, _encoding, callback) {\n await transaction?.attach(async () => {\n const { type, id, data } = entity;\n const { create, getDeepPopulateComponentLikeQuery } = query(type);\n const contentType = strapi.getModel(type);\n\n try {\n const created = await create({\n data,\n populate: getDeepPopulateComponentLikeQuery(contentType, { select: 'id' }),\n select: 'id',\n });\n\n // Compute differences between original & new entities\n const diffs = json.diff(data, created);\n\n updateMappingTable(type, id, created.id);\n\n // For each difference found on an ID attribute,\n // update the mapping the table accordingly\n diffs.forEach((diff) => {\n if (diff.kind === 'modified' && last(diff.path) === 'id' && 'kind' in contentType) {\n const target = resolveComponentUID({ paths: diff.path, data, contentType, strapi });\n\n // If no type is found for the given path, then ignore the diff\n if (!target) {\n return;\n }\n\n const [oldID, newID] = diff.values as [number, number];\n\n updateMappingTable(target, oldID, newID);\n }\n });\n } catch (e) {\n if (e instanceof Error) {\n return callback(e);\n }\n\n return callback(new ProviderTransferError(`Failed to create \"${type}\" (${id})`));\n }\n\n return callback(null);\n });\n },\n });\n};\n"],"names":["createEntitiesWriteStream","options","strapi","updateMappingTable","transaction","query","queries","Writable","objectMode","write","entity","_encoding","callback","attach","type","id","data","create","getDeepPopulateComponentLikeQuery","contentType","getModel","created","populate","select","diffs","json","forEach","diff","kind","last","path","target","resolveComponentUID","paths","oldID","newID","values","e","Error","ProviderTransferError"],"mappings":";;;;;;;;;AAqBO,MAAMA,4BAA4B,CAACC,OAAAA,GAAAA;AACxC,IAAA,MAAM,EAAEC,MAAM,EAAEC,kBAAkB,EAAEC,WAAW,EAAE,GAAGH,OAAAA;AACpD,IAAA,MAAMI,KAAQC,GAAAA,iBAAgC,CAACJ,MAAAA,CAAAA;AAE/C,IAAA,OAAO,IAAIK,QAAS,CAAA;QAClBC,UAAY,EAAA,IAAA;AAEZ,QAAA,MAAMC,KAAMC,CAAAA,CAAAA,MAAe,EAAEC,SAAS,EAAEC,QAAQ,EAAA;AAC9C,YAAA,MAAMR,aAAaS,MAAO,CAAA,UAAA;AACxB,gBAAA,MAAM,EAAEC,IAAI,EAAEC,EAAE,EAAEC,IAAI,EAAE,GAAGN,MAAAA;AAC3B,gBAAA,MAAM,EAAEO,MAAM,EAAEC,iCAAiC,EAAE,GAAGb,KAAMS,CAAAA,IAAAA,CAAAA;gBAC5D,MAAMK,WAAAA,GAAcjB,MAAOkB,CAAAA,QAAQ,CAACN,IAAAA,CAAAA;gBAEpC,IAAI;oBACF,MAAMO,OAAAA,GAAU,MAAMJ,MAAO,CAAA;AAC3BD,wBAAAA,IAAAA;AACAM,wBAAAA,QAAAA,EAAUJ,kCAAkCC,WAAa,EAAA;4BAAEI,MAAQ,EAAA;AAAK,yBAAA,CAAA;wBACxEA,MAAQ,EAAA;AACV,qBAAA,CAAA;;AAGA,oBAAA,MAAMC,KAAQC,GAAAA,IAAS,CAACT,IAAMK,EAAAA,OAAAA,CAAAA;oBAE9BlB,kBAAmBW,CAAAA,IAAAA,EAAMC,EAAIM,EAAAA,OAAAA,CAAQN,EAAE,CAAA;;;oBAIvCS,KAAME,CAAAA,OAAO,CAAC,CAACC,IAAAA,GAAAA;wBACb,IAAIA,IAAAA,CAAKC,IAAI,KAAK,UAAcC,IAAAA,IAAAA,CAAKF,KAAKG,IAAI,CAAA,KAAM,IAAQ,IAAA,MAAA,IAAUX,WAAa,EAAA;AACjF,4BAAA,MAAMY,SAASC,mBAAoB,CAAA;AAAEC,gCAAAA,KAAAA,EAAON,KAAKG,IAAI;AAAEd,gCAAAA,IAAAA;AAAMG,gCAAAA,WAAAA;AAAajB,gCAAAA;AAAO,6BAAA,CAAA;;AAGjF,4BAAA,IAAI,CAAC6B,MAAQ,EAAA;AACX,gCAAA;AACF;AAEA,4BAAA,MAAM,CAACG,KAAAA,EAAOC,KAAM,CAAA,GAAGR,KAAKS,MAAM;AAElCjC,4BAAAA,kBAAAA,CAAmB4B,QAAQG,KAAOC,EAAAA,KAAAA,CAAAA;AACpC;AACF,qBAAA,CAAA;AACF,iBAAA,CAAE,OAAOE,CAAG,EAAA;AACV,oBAAA,IAAIA,aAAaC,KAAO,EAAA;AACtB,wBAAA,OAAO1B,QAASyB,CAAAA,CAAAA,CAAAA;AAClB;oBAEA,OAAOzB,QAAAA,CAAS,IAAI2B,qBAAAA,CAAsB,CAAC,kBAAkB,EAAEzB,IAAAA,CAAK,GAAG,EAAEC,EAAG,CAAA,CAAC,CAAC,CAAA,CAAA;AAChF;AAEA,gBAAA,OAAOH,QAAS,CAAA,IAAA,CAAA;AAClB,aAAA,CAAA;AACF;AACF,KAAA,CAAA;AACF;;;;"}

View File

@@ -0,0 +1,33 @@
import type { Core, Struct } from '@strapi/types';
export interface IRestoreOptions {
assets?: boolean;
configuration?: {
webhook?: boolean;
coreStore?: boolean;
};
entities?: {
include?: string[];
exclude?: string[];
filters?: ((contentType: Struct.ContentTypeSchema) => boolean)[];
params?: {
[uid: string]: unknown;
};
};
}
interface IDeleteResults {
count: number;
aggregate: {
[uid: string]: {
count: number;
};
};
}
export declare const deleteRecords: (strapi: Core.Strapi, options: IRestoreOptions) => Promise<{
count: number;
entities: IDeleteResults;
configuration: IDeleteResults;
}>;
export * from './entities';
export * from './configuration';
export * from './links';
//# sourceMappingURL=index.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAO,MAAM,EAAE,MAAM,eAAe,CAAC;AAKvD,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE;QACd,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;IACF,QAAQ,CAAC,EAAE;QACT,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,iBAAiB,KAAK,OAAO,CAAC,EAAE,CAAC;QACjE,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,CAAC;KACrC,CAAC;CACH;AAED,UAAU,cAAc;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;CACjD;AAED,eAAO,MAAM,aAAa,WAAkB,KAAK,MAAM,WAAW,eAAe;;;;EAShF,CAAC;AAsIF,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,SAAS,CAAC"}

View File

@@ -0,0 +1,126 @@
'use strict';
var providers = require('../../../../../errors/providers.js');
var entity = require('../../../../queries/entity.js');
require('lodash/fp');
require('stream');
require('crypto');
require('events');
require('lodash');
require('@strapi/utils');
var configuration = require('./configuration.js');
const deleteRecords = async (strapi, options)=>{
const entities = await deleteEntitiesRecords(strapi, options);
const configuration = await deleteConfigurationRecords(strapi, options);
return {
count: entities.count + configuration.count,
entities,
configuration
};
};
const deleteEntitiesRecords = async (strapi, options = {})=>{
const { entities } = options;
const models = strapi.get('models').get();
const contentTypes = Object.values(strapi.contentTypes);
const contentTypesToClear = contentTypes.filter((contentType)=>{
let removeThisContentType = true;
// include means "only include these types" so if it's not in here, it's not being included
if (entities?.include) {
removeThisContentType = entities.include.includes(contentType.uid);
}
// if something is excluded, remove it. But lack of being excluded doesn't mean it's kept
if (entities?.exclude && entities.exclude.includes(contentType.uid)) {
removeThisContentType = false;
}
if (entities?.filters) {
removeThisContentType = entities.filters.every((filter)=>filter(contentType));
}
return removeThisContentType;
}).map((contentType)=>contentType.uid);
const modelsToClear = models.filter((model)=>{
if (contentTypesToClear.includes(model.uid)) {
return false;
}
let removeThisModel = true;
// include means "only include these types" so if it's not in here, it's not being included
if (entities?.include) {
removeThisModel = entities.include.includes(model.uid);
}
// if something is excluded, remove it. But lack of being excluded doesn't mean it's kept
if (entities?.exclude && entities.exclude.includes(model.uid)) {
removeThisModel = false;
}
return removeThisModel;
}).map((model)=>model.uid);
const [results, updateResults] = useResults([
...contentTypesToClear,
...modelsToClear
]);
const contentTypeQuery = entity.createEntityQuery(strapi);
const contentTypePromises = contentTypesToClear.map(async (uid)=>{
const result = await contentTypeQuery(uid).deleteMany(entities?.params);
if (result) {
updateResults(result.count || 0, uid);
}
});
const modelsPromises = modelsToClear.map(async (uid)=>{
const result = await strapi.db.query(uid).deleteMany({});
if (result) {
updateResults(result.count || 0, uid);
}
});
await Promise.all([
...contentTypePromises,
...modelsPromises
]);
return results;
};
const deleteConfigurationRecords = async (strapi, options = {})=>{
const { coreStore = true, webhook = true } = options?.configuration ?? {};
const models = [];
if (coreStore) {
models.push('strapi::core-store');
}
if (webhook) {
models.push('strapi::webhook');
}
const [results, updateResults] = useResults(models);
const deletePromises = models.map(async (uid)=>{
const result = await strapi.db.query(uid).deleteMany({});
if (result) {
updateResults(result.count, uid);
}
});
await Promise.all(deletePromises);
return results;
};
const useResults = (keys)=>{
const results = {
count: 0,
aggregate: keys.reduce((acc, key)=>({
...acc,
[key]: {
count: 0
}
}), {})
};
const update = (count, key)=>{
if (key) {
if (!(key in results.aggregate)) {
throw new providers.ProviderTransferError(`Unknown key "${key}" provided in results update`);
}
results.aggregate[key].count += count;
}
results.count += count;
};
return [
results,
update
];
};
exports.createConfigurationWriteStream = configuration.createConfigurationWriteStream;
exports.restoreConfigs = configuration.restoreConfigs;
exports.deleteRecords = deleteRecords;
//# sourceMappingURL=index.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,122 @@
import { ProviderTransferError } from '../../../../../errors/providers.mjs';
import { createEntityQuery } from '../../../../queries/entity.mjs';
import 'lodash/fp';
import 'stream';
import 'crypto';
import 'events';
import 'lodash';
import '@strapi/utils';
export { createConfigurationWriteStream, restoreConfigs } from './configuration.mjs';
const deleteRecords = async (strapi, options)=>{
const entities = await deleteEntitiesRecords(strapi, options);
const configuration = await deleteConfigurationRecords(strapi, options);
return {
count: entities.count + configuration.count,
entities,
configuration
};
};
const deleteEntitiesRecords = async (strapi, options = {})=>{
const { entities } = options;
const models = strapi.get('models').get();
const contentTypes = Object.values(strapi.contentTypes);
const contentTypesToClear = contentTypes.filter((contentType)=>{
let removeThisContentType = true;
// include means "only include these types" so if it's not in here, it's not being included
if (entities?.include) {
removeThisContentType = entities.include.includes(contentType.uid);
}
// if something is excluded, remove it. But lack of being excluded doesn't mean it's kept
if (entities?.exclude && entities.exclude.includes(contentType.uid)) {
removeThisContentType = false;
}
if (entities?.filters) {
removeThisContentType = entities.filters.every((filter)=>filter(contentType));
}
return removeThisContentType;
}).map((contentType)=>contentType.uid);
const modelsToClear = models.filter((model)=>{
if (contentTypesToClear.includes(model.uid)) {
return false;
}
let removeThisModel = true;
// include means "only include these types" so if it's not in here, it's not being included
if (entities?.include) {
removeThisModel = entities.include.includes(model.uid);
}
// if something is excluded, remove it. But lack of being excluded doesn't mean it's kept
if (entities?.exclude && entities.exclude.includes(model.uid)) {
removeThisModel = false;
}
return removeThisModel;
}).map((model)=>model.uid);
const [results, updateResults] = useResults([
...contentTypesToClear,
...modelsToClear
]);
const contentTypeQuery = createEntityQuery(strapi);
const contentTypePromises = contentTypesToClear.map(async (uid)=>{
const result = await contentTypeQuery(uid).deleteMany(entities?.params);
if (result) {
updateResults(result.count || 0, uid);
}
});
const modelsPromises = modelsToClear.map(async (uid)=>{
const result = await strapi.db.query(uid).deleteMany({});
if (result) {
updateResults(result.count || 0, uid);
}
});
await Promise.all([
...contentTypePromises,
...modelsPromises
]);
return results;
};
const deleteConfigurationRecords = async (strapi, options = {})=>{
const { coreStore = true, webhook = true } = options?.configuration ?? {};
const models = [];
if (coreStore) {
models.push('strapi::core-store');
}
if (webhook) {
models.push('strapi::webhook');
}
const [results, updateResults] = useResults(models);
const deletePromises = models.map(async (uid)=>{
const result = await strapi.db.query(uid).deleteMany({});
if (result) {
updateResults(result.count, uid);
}
});
await Promise.all(deletePromises);
return results;
};
const useResults = (keys)=>{
const results = {
count: 0,
aggregate: keys.reduce((acc, key)=>({
...acc,
[key]: {
count: 0
}
}), {})
};
const update = (count, key)=>{
if (key) {
if (!(key in results.aggregate)) {
throw new ProviderTransferError(`Unknown key "${key}" provided in results update`);
}
results.aggregate[key].count += count;
}
results.count += count;
};
return [
results,
update
];
};
export { deleteRecords };
//# sourceMappingURL=index.mjs.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,6 @@
/// <reference types="node" />
import { Writable } from 'stream';
import type { Core } from '@strapi/types';
import { Transaction } from '../../../../../../types';
export declare const createLinksWriteStream: (mapID: (uid: string, id: number) => number | undefined, strapi: Core.Strapi, transaction?: Transaction, onWarning?: (message: string) => void) => Writable;
//# sourceMappingURL=links.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"links.d.ts","sourceRoot":"","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/links.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAE1C,OAAO,EAAS,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAuB7D,eAAO,MAAM,sBAAsB,UAC1B,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,UAC9C,KAAK,MAAM,gBACL,WAAW,cACb,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,aAwCtC,CAAC"}

View File

@@ -0,0 +1,60 @@
'use strict';
var stream = require('stream');
var providers = require('../../../../../errors/providers.js');
var link = require('../../../../queries/link.js');
const isErrorWithCode = (error)=>{
return error && typeof error.code === 'string';
};
const isForeignKeyConstraintError = (e)=>{
const MYSQL_FK_ERROR_CODES = [
'1452',
'1557',
'1216',
'1217',
'1451'
];
const POSTGRES_FK_ERROR_CODE = '23503';
const SQLITE_FK_ERROR_CODE = 'SQLITE_CONSTRAINT_FOREIGNKEY';
if (isErrorWithCode(e) && e.code) {
return [
SQLITE_FK_ERROR_CODE,
POSTGRES_FK_ERROR_CODE,
...MYSQL_FK_ERROR_CODES
].includes(e.code);
}
return e.message.toLowerCase().includes('foreign key constraint');
};
const createLinksWriteStream = (mapID, strapi, transaction, onWarning)=>{
return new stream.Writable({
objectMode: true,
async write (link$1, _encoding, callback) {
await transaction?.attach(async (trx)=>{
const { left, right } = link$1;
const query = link.createLinkQuery(strapi, trx);
const originalLeftRef = left.ref;
const originalRightRef = right.ref;
// Map IDs if needed
left.ref = mapID(left.type, originalLeftRef) ?? originalLeftRef;
right.ref = mapID(right.type, originalRightRef) ?? originalRightRef;
try {
await query().insert(link$1);
} catch (e) {
if (e instanceof Error) {
if (isForeignKeyConstraintError(e)) {
onWarning?.(`Skipping link ${left.type}:${originalLeftRef} -> ${right.type}:${originalRightRef} due to a foreign key constraint.`);
return callback(null);
}
return callback(e);
}
return callback(new providers.ProviderTransferError(`An error happened while trying to import a ${left.type} link.`));
}
callback(null);
});
}
});
};
exports.createLinksWriteStream = createLinksWriteStream;
//# sourceMappingURL=links.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"links.js","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/links.ts"],"sourcesContent":["import { Writable } from 'stream';\nimport type { Core } from '@strapi/types';\nimport { ProviderTransferError } from '../../../../../errors/providers';\nimport { ILink, Transaction } from '../../../../../../types';\nimport { createLinkQuery } from '../../../../queries/link';\n\ninterface ErrorWithCode extends Error {\n code: string;\n}\n\nconst isErrorWithCode = (error: any): error is ErrorWithCode => {\n return error && typeof error.code === 'string';\n};\n\nconst isForeignKeyConstraintError = (e: Error) => {\n const MYSQL_FK_ERROR_CODES = ['1452', '1557', '1216', '1217', '1451'];\n const POSTGRES_FK_ERROR_CODE = '23503';\n const SQLITE_FK_ERROR_CODE = 'SQLITE_CONSTRAINT_FOREIGNKEY';\n\n if (isErrorWithCode(e) && e.code) {\n return [SQLITE_FK_ERROR_CODE, POSTGRES_FK_ERROR_CODE, ...MYSQL_FK_ERROR_CODES].includes(e.code);\n }\n\n return e.message.toLowerCase().includes('foreign key constraint');\n};\n\nexport const createLinksWriteStream = (\n mapID: (uid: string, id: number) => number | undefined,\n strapi: Core.Strapi,\n transaction?: Transaction,\n onWarning?: (message: string) => void\n) => {\n return new Writable({\n objectMode: true,\n async write(link: ILink, _encoding, callback) {\n await transaction?.attach(async (trx) => {\n const { left, right } = link;\n const query = createLinkQuery(strapi, trx);\n\n const originalLeftRef = left.ref;\n const originalRightRef = right.ref;\n\n // Map IDs if needed\n left.ref = mapID(left.type, originalLeftRef) ?? originalLeftRef;\n right.ref = mapID(right.type, originalRightRef) ?? originalRightRef;\n\n try {\n await query().insert(link);\n } catch (e) {\n if (e instanceof Error) {\n if (isForeignKeyConstraintError(e)) {\n onWarning?.(\n `Skipping link ${left.type}:${originalLeftRef} -> ${right.type}:${originalRightRef} due to a foreign key constraint.`\n );\n return callback(null);\n }\n return callback(e);\n }\n\n return callback(\n new ProviderTransferError(\n `An error happened while trying to import a ${left.type} link.`\n )\n );\n }\n\n callback(null);\n });\n },\n });\n};\n"],"names":["isErrorWithCode","error","code","isForeignKeyConstraintError","e","MYSQL_FK_ERROR_CODES","POSTGRES_FK_ERROR_CODE","SQLITE_FK_ERROR_CODE","includes","message","toLowerCase","createLinksWriteStream","mapID","strapi","transaction","onWarning","Writable","objectMode","write","link","_encoding","callback","attach","trx","left","right","query","createLinkQuery","originalLeftRef","ref","originalRightRef","type","insert","Error","ProviderTransferError"],"mappings":";;;;;;AAUA,MAAMA,kBAAkB,CAACC,KAAAA,GAAAA;AACvB,IAAA,OAAOA,KAAS,IAAA,OAAOA,KAAMC,CAAAA,IAAI,KAAK,QAAA;AACxC,CAAA;AAEA,MAAMC,8BAA8B,CAACC,CAAAA,GAAAA;AACnC,IAAA,MAAMC,oBAAuB,GAAA;AAAC,QAAA,MAAA;AAAQ,QAAA,MAAA;AAAQ,QAAA,MAAA;AAAQ,QAAA,MAAA;AAAQ,QAAA;AAAO,KAAA;AACrE,IAAA,MAAMC,sBAAyB,GAAA,OAAA;AAC/B,IAAA,MAAMC,oBAAuB,GAAA,8BAAA;AAE7B,IAAA,IAAIP,eAAgBI,CAAAA,CAAAA,CAAAA,IAAMA,CAAEF,CAAAA,IAAI,EAAE;QAChC,OAAO;AAACK,YAAAA,oBAAAA;AAAsBD,YAAAA,sBAAAA;AAA2BD,YAAAA,GAAAA;SAAqB,CAACG,QAAQ,CAACJ,CAAAA,CAAEF,IAAI,CAAA;AAChG;AAEA,IAAA,OAAOE,EAAEK,OAAO,CAACC,WAAW,EAAA,CAAGF,QAAQ,CAAC,wBAAA,CAAA;AAC1C,CAAA;AAEaG,MAAAA,sBAAAA,GAAyB,CACpCC,KAAAA,EACAC,QACAC,WACAC,EAAAA,SAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,eAAS,CAAA;QAClBC,UAAY,EAAA,IAAA;AACZ,QAAA,MAAMC,KAAMC,CAAAA,CAAAA,MAAW,EAAEC,SAAS,EAAEC,QAAQ,EAAA;YAC1C,MAAMP,WAAAA,EAAaQ,OAAO,OAAOC,GAAAA,GAAAA;AAC/B,gBAAA,MAAM,EAAEC,IAAI,EAAEC,KAAK,EAAE,GAAGN,MAAAA;gBACxB,MAAMO,KAAAA,GAAQC,qBAAgBd,MAAQU,EAAAA,GAAAA,CAAAA;gBAEtC,MAAMK,eAAAA,GAAkBJ,KAAKK,GAAG;gBAChC,MAAMC,gBAAAA,GAAmBL,MAAMI,GAAG;;AAGlCL,gBAAAA,IAAAA,CAAKK,GAAG,GAAGjB,KAAAA,CAAMY,IAAKO,CAAAA,IAAI,EAAEH,eAAoBA,CAAAA,IAAAA,eAAAA;AAChDH,gBAAAA,KAAAA,CAAMI,GAAG,GAAGjB,KAAAA,CAAMa,KAAMM,CAAAA,IAAI,EAAED,gBAAqBA,CAAAA,IAAAA,gBAAAA;gBAEnD,IAAI;oBACF,MAAMJ,KAAAA,EAAAA,CAAQM,MAAM,CAACb,MAAAA,CAAAA;AACvB,iBAAA,CAAE,OAAOf,CAAG,EAAA;AACV,oBAAA,IAAIA,aAAa6B,KAAO,EAAA;AACtB,wBAAA,IAAI9B,4BAA4BC,CAAI,CAAA,EAAA;AAClCW,4BAAAA,SAAAA,GACE,CAAC,cAAc,EAAES,KAAKO,IAAI,CAAC,CAAC,EAAEH,eAAAA,CAAgB,IAAI,EAAEH,MAAMM,IAAI,CAAC,CAAC,EAAED,gBAAAA,CAAiB,iCAAiC,CAAC,CAAA;AAEvH,4BAAA,OAAOT,QAAS,CAAA,IAAA,CAAA;AAClB;AACA,wBAAA,OAAOA,QAASjB,CAAAA,CAAAA,CAAAA;AAClB;oBAEA,OAAOiB,QAAAA,CACL,IAAIa,+BAAAA,CACF,CAAC,2CAA2C,EAAEV,IAAKO,CAAAA,IAAI,CAAC,MAAM,CAAC,CAAA,CAAA;AAGrE;gBAEAV,QAAS,CAAA,IAAA,CAAA;AACX,aAAA,CAAA;AACF;AACF,KAAA,CAAA;AACF;;;;"}

View File

@@ -0,0 +1,58 @@
import { Writable } from 'stream';
import { ProviderTransferError } from '../../../../../errors/providers.mjs';
import { createLinkQuery } from '../../../../queries/link.mjs';
const isErrorWithCode = (error)=>{
return error && typeof error.code === 'string';
};
const isForeignKeyConstraintError = (e)=>{
const MYSQL_FK_ERROR_CODES = [
'1452',
'1557',
'1216',
'1217',
'1451'
];
const POSTGRES_FK_ERROR_CODE = '23503';
const SQLITE_FK_ERROR_CODE = 'SQLITE_CONSTRAINT_FOREIGNKEY';
if (isErrorWithCode(e) && e.code) {
return [
SQLITE_FK_ERROR_CODE,
POSTGRES_FK_ERROR_CODE,
...MYSQL_FK_ERROR_CODES
].includes(e.code);
}
return e.message.toLowerCase().includes('foreign key constraint');
};
const createLinksWriteStream = (mapID, strapi, transaction, onWarning)=>{
return new Writable({
objectMode: true,
async write (link, _encoding, callback) {
await transaction?.attach(async (trx)=>{
const { left, right } = link;
const query = createLinkQuery(strapi, trx);
const originalLeftRef = left.ref;
const originalRightRef = right.ref;
// Map IDs if needed
left.ref = mapID(left.type, originalLeftRef) ?? originalLeftRef;
right.ref = mapID(right.type, originalRightRef) ?? originalRightRef;
try {
await query().insert(link);
} catch (e) {
if (e instanceof Error) {
if (isForeignKeyConstraintError(e)) {
onWarning?.(`Skipping link ${left.type}:${originalLeftRef} -> ${right.type}:${originalRightRef} due to a foreign key constraint.`);
return callback(null);
}
return callback(e);
}
return callback(new ProviderTransferError(`An error happened while trying to import a ${left.type} link.`));
}
callback(null);
});
}
});
};
export { createLinksWriteStream };
//# sourceMappingURL=links.mjs.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"links.mjs","sources":["../../../../../../src/strapi/providers/local-destination/strategies/restore/links.ts"],"sourcesContent":["import { Writable } from 'stream';\nimport type { Core } from '@strapi/types';\nimport { ProviderTransferError } from '../../../../../errors/providers';\nimport { ILink, Transaction } from '../../../../../../types';\nimport { createLinkQuery } from '../../../../queries/link';\n\ninterface ErrorWithCode extends Error {\n code: string;\n}\n\nconst isErrorWithCode = (error: any): error is ErrorWithCode => {\n return error && typeof error.code === 'string';\n};\n\nconst isForeignKeyConstraintError = (e: Error) => {\n const MYSQL_FK_ERROR_CODES = ['1452', '1557', '1216', '1217', '1451'];\n const POSTGRES_FK_ERROR_CODE = '23503';\n const SQLITE_FK_ERROR_CODE = 'SQLITE_CONSTRAINT_FOREIGNKEY';\n\n if (isErrorWithCode(e) && e.code) {\n return [SQLITE_FK_ERROR_CODE, POSTGRES_FK_ERROR_CODE, ...MYSQL_FK_ERROR_CODES].includes(e.code);\n }\n\n return e.message.toLowerCase().includes('foreign key constraint');\n};\n\nexport const createLinksWriteStream = (\n mapID: (uid: string, id: number) => number | undefined,\n strapi: Core.Strapi,\n transaction?: Transaction,\n onWarning?: (message: string) => void\n) => {\n return new Writable({\n objectMode: true,\n async write(link: ILink, _encoding, callback) {\n await transaction?.attach(async (trx) => {\n const { left, right } = link;\n const query = createLinkQuery(strapi, trx);\n\n const originalLeftRef = left.ref;\n const originalRightRef = right.ref;\n\n // Map IDs if needed\n left.ref = mapID(left.type, originalLeftRef) ?? originalLeftRef;\n right.ref = mapID(right.type, originalRightRef) ?? originalRightRef;\n\n try {\n await query().insert(link);\n } catch (e) {\n if (e instanceof Error) {\n if (isForeignKeyConstraintError(e)) {\n onWarning?.(\n `Skipping link ${left.type}:${originalLeftRef} -> ${right.type}:${originalRightRef} due to a foreign key constraint.`\n );\n return callback(null);\n }\n return callback(e);\n }\n\n return callback(\n new ProviderTransferError(\n `An error happened while trying to import a ${left.type} link.`\n )\n );\n }\n\n callback(null);\n });\n },\n });\n};\n"],"names":["isErrorWithCode","error","code","isForeignKeyConstraintError","e","MYSQL_FK_ERROR_CODES","POSTGRES_FK_ERROR_CODE","SQLITE_FK_ERROR_CODE","includes","message","toLowerCase","createLinksWriteStream","mapID","strapi","transaction","onWarning","Writable","objectMode","write","link","_encoding","callback","attach","trx","left","right","query","createLinkQuery","originalLeftRef","ref","originalRightRef","type","insert","Error","ProviderTransferError"],"mappings":";;;;AAUA,MAAMA,kBAAkB,CAACC,KAAAA,GAAAA;AACvB,IAAA,OAAOA,KAAS,IAAA,OAAOA,KAAMC,CAAAA,IAAI,KAAK,QAAA;AACxC,CAAA;AAEA,MAAMC,8BAA8B,CAACC,CAAAA,GAAAA;AACnC,IAAA,MAAMC,oBAAuB,GAAA;AAAC,QAAA,MAAA;AAAQ,QAAA,MAAA;AAAQ,QAAA,MAAA;AAAQ,QAAA,MAAA;AAAQ,QAAA;AAAO,KAAA;AACrE,IAAA,MAAMC,sBAAyB,GAAA,OAAA;AAC/B,IAAA,MAAMC,oBAAuB,GAAA,8BAAA;AAE7B,IAAA,IAAIP,eAAgBI,CAAAA,CAAAA,CAAAA,IAAMA,CAAEF,CAAAA,IAAI,EAAE;QAChC,OAAO;AAACK,YAAAA,oBAAAA;AAAsBD,YAAAA,sBAAAA;AAA2BD,YAAAA,GAAAA;SAAqB,CAACG,QAAQ,CAACJ,CAAAA,CAAEF,IAAI,CAAA;AAChG;AAEA,IAAA,OAAOE,EAAEK,OAAO,CAACC,WAAW,EAAA,CAAGF,QAAQ,CAAC,wBAAA,CAAA;AAC1C,CAAA;AAEaG,MAAAA,sBAAAA,GAAyB,CACpCC,KAAAA,EACAC,QACAC,WACAC,EAAAA,SAAAA,GAAAA;AAEA,IAAA,OAAO,IAAIC,QAAS,CAAA;QAClBC,UAAY,EAAA,IAAA;AACZ,QAAA,MAAMC,KAAMC,CAAAA,CAAAA,IAAW,EAAEC,SAAS,EAAEC,QAAQ,EAAA;YAC1C,MAAMP,WAAAA,EAAaQ,OAAO,OAAOC,GAAAA,GAAAA;AAC/B,gBAAA,MAAM,EAAEC,IAAI,EAAEC,KAAK,EAAE,GAAGN,IAAAA;gBACxB,MAAMO,KAAAA,GAAQC,gBAAgBd,MAAQU,EAAAA,GAAAA,CAAAA;gBAEtC,MAAMK,eAAAA,GAAkBJ,KAAKK,GAAG;gBAChC,MAAMC,gBAAAA,GAAmBL,MAAMI,GAAG;;AAGlCL,gBAAAA,IAAAA,CAAKK,GAAG,GAAGjB,KAAAA,CAAMY,IAAKO,CAAAA,IAAI,EAAEH,eAAoBA,CAAAA,IAAAA,eAAAA;AAChDH,gBAAAA,KAAAA,CAAMI,GAAG,GAAGjB,KAAAA,CAAMa,KAAMM,CAAAA,IAAI,EAAED,gBAAqBA,CAAAA,IAAAA,gBAAAA;gBAEnD,IAAI;oBACF,MAAMJ,KAAAA,EAAAA,CAAQM,MAAM,CAACb,IAAAA,CAAAA;AACvB,iBAAA,CAAE,OAAOf,CAAG,EAAA;AACV,oBAAA,IAAIA,aAAa6B,KAAO,EAAA;AACtB,wBAAA,IAAI9B,4BAA4BC,CAAI,CAAA,EAAA;AAClCW,4BAAAA,SAAAA,GACE,CAAC,cAAc,EAAES,KAAKO,IAAI,CAAC,CAAC,EAAEH,eAAAA,CAAgB,IAAI,EAAEH,MAAMM,IAAI,CAAC,CAAC,EAAED,gBAAAA,CAAiB,iCAAiC,CAAC,CAAA;AAEvH,4BAAA,OAAOT,QAAS,CAAA,IAAA,CAAA;AAClB;AACA,wBAAA,OAAOA,QAASjB,CAAAA,CAAAA,CAAAA;AAClB;oBAEA,OAAOiB,QAAAA,CACL,IAAIa,qBAAAA,CACF,CAAC,2CAA2C,EAAEV,IAAKO,CAAAA,IAAI,CAAC,MAAM,CAAC,CAAA,CAAA;AAGrE;gBAEAV,QAAS,CAAA,IAAA,CAAA;AACX,aAAA,CAAA;AACF;AACF,KAAA,CAAA;AACF;;;;"}

View File

@@ -0,0 +1,8 @@
/// <reference types="node" />
import { Duplex } from 'stream';
import type { Core } from '@strapi/types';
/**
* Generate and consume assets streams in order to stream each file individually
*/
export declare const createAssetsStream: (strapi: Core.Strapi) => Duplex;
//# sourceMappingURL=assets.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"assets.d.ts","sourceRoot":"","sources":["../../../../src/strapi/providers/local-source/assets.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,EAAyB,MAAM,QAAQ,CAAC;AAGvD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AA0F1C;;GAEG;AACH,eAAO,MAAM,kBAAkB,WAAY,KAAK,MAAM,KAAG,MAiDxD,CAAC"}

View File

@@ -0,0 +1,123 @@
'use strict';
var path = require('path');
var stream = require('stream');
var fse = require('fs-extra');
function getFileStream(filepath, strapi1, isLocal = false) {
if (isLocal) {
// Todo: handle errors
return fse.createReadStream(filepath);
}
const readableStream = new stream.PassThrough();
// fetch the image from remote url and stream it
strapi1.fetch(filepath).then((res)=>{
if (res.status !== 200) {
readableStream.emit('error', new Error(`Request failed with status code ${res.status}`));
return;
}
if (res.body) {
// pipe the image data
stream.Readable.fromWeb(res.body).pipe(readableStream);
} else {
readableStream.emit('error', new Error('Empty data found for file'));
}
}).catch((error)=>{
readableStream.emit('error', error);
});
return readableStream;
}
function getFileStats(filepath, strapi1, isLocal = false) {
if (isLocal) {
return fse.stat(filepath);
}
return new Promise((resolve, reject)=>{
strapi1.fetch(filepath).then((res)=>{
if (res.status !== 200) {
reject(new Error(`Request failed with status code ${res.status}`));
return;
}
const contentLength = res.headers.get('content-length');
const stats = {
size: contentLength ? parseInt(contentLength, 10) : 0
};
resolve(stats);
}).catch((error)=>{
reject(error);
});
});
}
async function signFile(file) {
const { provider } = strapi.plugins.upload;
const { provider: providerName } = strapi.config.get('plugin.upload');
const isPrivate = await provider.isPrivate();
if (file?.provider === providerName && isPrivate) {
const signUrl = async (file)=>{
const signedUrl = await provider.getSignedUrl(file);
file.url = signedUrl.url;
};
// Sign the original file
await signUrl(file);
// Sign each file format
if (file.formats) {
for (const format of Object.keys(file.formats)){
await signUrl(file.formats[format]);
}
}
}
}
/**
* Generate and consume assets streams in order to stream each file individually
*/ const createAssetsStream = (strapi1)=>{
const generator = async function*() {
const stream = strapi1.db.queryBuilder('plugin::upload.file')// Create a query builder instance (default type is 'select')
// Fetch all columns
.select('*')// Get a readable stream
.stream();
for await (const file of stream){
const isLocalProvider = file.provider === 'local';
if (!isLocalProvider) {
await signFile(file);
}
const filepath = isLocalProvider ? path.join(strapi1.dirs.static.public, file.url) : file.url;
const stats = await getFileStats(filepath, strapi1, isLocalProvider);
const stream = getFileStream(filepath, strapi1, isLocalProvider);
yield {
metadata: file,
filepath,
filename: file.hash + file.ext,
stream,
stats: {
size: stats.size
}
};
if (file.formats) {
for (const format of Object.keys(file.formats)){
const fileFormat = file.formats[format];
const fileFormatFilepath = isLocalProvider ? path.join(strapi1.dirs.static.public, fileFormat.url) : fileFormat.url;
const fileFormatStats = await getFileStats(fileFormatFilepath, strapi1, isLocalProvider);
const fileFormatStream = getFileStream(fileFormatFilepath, strapi1, isLocalProvider);
const metadata = {
...fileFormat,
type: format,
id: file.id,
mainHash: file.hash
};
yield {
metadata,
filepath: fileFormatFilepath,
filename: fileFormat.hash + fileFormat.ext,
stream: fileFormatStream,
stats: {
size: fileFormatStats.size
}
};
}
}
}
};
return stream.Duplex.from(generator());
};
exports.createAssetsStream = createAssetsStream;
//# sourceMappingURL=assets.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,121 @@
import { join } from 'path';
import { Duplex, PassThrough, Readable } from 'stream';
import { createReadStream, stat } from 'fs-extra';
function getFileStream(filepath, strapi1, isLocal = false) {
if (isLocal) {
// Todo: handle errors
return createReadStream(filepath);
}
const readableStream = new PassThrough();
// fetch the image from remote url and stream it
strapi1.fetch(filepath).then((res)=>{
if (res.status !== 200) {
readableStream.emit('error', new Error(`Request failed with status code ${res.status}`));
return;
}
if (res.body) {
// pipe the image data
Readable.fromWeb(res.body).pipe(readableStream);
} else {
readableStream.emit('error', new Error('Empty data found for file'));
}
}).catch((error)=>{
readableStream.emit('error', error);
});
return readableStream;
}
function getFileStats(filepath, strapi1, isLocal = false) {
if (isLocal) {
return stat(filepath);
}
return new Promise((resolve, reject)=>{
strapi1.fetch(filepath).then((res)=>{
if (res.status !== 200) {
reject(new Error(`Request failed with status code ${res.status}`));
return;
}
const contentLength = res.headers.get('content-length');
const stats = {
size: contentLength ? parseInt(contentLength, 10) : 0
};
resolve(stats);
}).catch((error)=>{
reject(error);
});
});
}
async function signFile(file) {
const { provider } = strapi.plugins.upload;
const { provider: providerName } = strapi.config.get('plugin.upload');
const isPrivate = await provider.isPrivate();
if (file?.provider === providerName && isPrivate) {
const signUrl = async (file)=>{
const signedUrl = await provider.getSignedUrl(file);
file.url = signedUrl.url;
};
// Sign the original file
await signUrl(file);
// Sign each file format
if (file.formats) {
for (const format of Object.keys(file.formats)){
await signUrl(file.formats[format]);
}
}
}
}
/**
* Generate and consume assets streams in order to stream each file individually
*/ const createAssetsStream = (strapi1)=>{
const generator = async function*() {
const stream = strapi1.db.queryBuilder('plugin::upload.file')// Create a query builder instance (default type is 'select')
// Fetch all columns
.select('*')// Get a readable stream
.stream();
for await (const file of stream){
const isLocalProvider = file.provider === 'local';
if (!isLocalProvider) {
await signFile(file);
}
const filepath = isLocalProvider ? join(strapi1.dirs.static.public, file.url) : file.url;
const stats = await getFileStats(filepath, strapi1, isLocalProvider);
const stream = getFileStream(filepath, strapi1, isLocalProvider);
yield {
metadata: file,
filepath,
filename: file.hash + file.ext,
stream,
stats: {
size: stats.size
}
};
if (file.formats) {
for (const format of Object.keys(file.formats)){
const fileFormat = file.formats[format];
const fileFormatFilepath = isLocalProvider ? join(strapi1.dirs.static.public, fileFormat.url) : fileFormat.url;
const fileFormatStats = await getFileStats(fileFormatFilepath, strapi1, isLocalProvider);
const fileFormatStream = getFileStream(fileFormatFilepath, strapi1, isLocalProvider);
const metadata = {
...fileFormat,
type: format,
id: file.id,
mainHash: file.hash
};
yield {
metadata,
filepath: fileFormatFilepath,
filename: fileFormat.hash + fileFormat.ext,
stream: fileFormatStream,
stats: {
size: fileFormatStats.size
}
};
}
}
}
};
return Duplex.from(generator());
};
export { createAssetsStream };
//# sourceMappingURL=assets.mjs.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,8 @@
/// <reference types="node" />
import { Readable } from 'stream';
import type { Core } from '@strapi/types';
/**
* Create a readable stream that export the Strapi app configuration
*/
export declare const createConfigurationStream: (strapi: Core.Strapi) => Readable;
//# sourceMappingURL=configuration.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../../../../src/strapi/providers/local-source/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAGlC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAI1C;;GAEG;AACH,eAAO,MAAM,yBAAyB,WAAY,KAAK,MAAM,KAAG,QAyB/D,CAAC"}

View File

@@ -0,0 +1,39 @@
'use strict';
var stream = require('stream');
var streamChain = require('stream-chain');
var fp = require('lodash/fp');
/**
* Create a readable stream that export the Strapi app configuration
*/ const createConfigurationStream = (strapi)=>{
return stream.Readable.from(async function* configurationGenerator() {
// Core Store
const coreStoreStream = streamChain.chain([
strapi.db.queryBuilder('strapi::core-store').stream(),
(data)=>fp.set('value', JSON.parse(data.value), data),
wrapConfigurationItem('core-store')
]);
// Webhook
const webhooksStream = streamChain.chain([
strapi.db.queryBuilder('strapi::webhook').stream(),
wrapConfigurationItem('webhook')
]);
const streams = [
coreStoreStream,
webhooksStream
];
for (const stream of streams){
for await (const item of stream){
yield item;
}
}
}());
};
const wrapConfigurationItem = (type)=>(value)=>({
type,
value
});
exports.createConfigurationStream = createConfigurationStream;
//# sourceMappingURL=configuration.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"configuration.js","sources":["../../../../src/strapi/providers/local-source/configuration.ts"],"sourcesContent":["import { Readable } from 'stream';\nimport { chain } from 'stream-chain';\nimport { set } from 'lodash/fp';\nimport type { Core } from '@strapi/types';\n\nimport type { IConfiguration } from '../../../../types';\n\n/**\n * Create a readable stream that export the Strapi app configuration\n */\nexport const createConfigurationStream = (strapi: Core.Strapi): Readable => {\n return Readable.from(\n (async function* configurationGenerator(): AsyncGenerator<IConfiguration> {\n // Core Store\n const coreStoreStream = chain([\n strapi.db.queryBuilder('strapi::core-store').stream(),\n (data) => set('value', JSON.parse(data.value), data),\n wrapConfigurationItem('core-store'),\n ]);\n\n // Webhook\n const webhooksStream = chain([\n strapi.db.queryBuilder('strapi::webhook').stream(),\n wrapConfigurationItem('webhook'),\n ]);\n\n const streams = [coreStoreStream, webhooksStream];\n\n for (const stream of streams) {\n for await (const item of stream) {\n yield item;\n }\n }\n })()\n );\n};\n\nconst wrapConfigurationItem = (type: 'core-store' | 'webhook') => (value: unknown) => ({\n type,\n value,\n});\n"],"names":["createConfigurationStream","strapi","Readable","from","configurationGenerator","coreStoreStream","chain","db","queryBuilder","stream","data","set","JSON","parse","value","wrapConfigurationItem","webhooksStream","streams","item","type"],"mappings":";;;;;;AAOA;;IAGaA,MAAAA,yBAAAA,GAA4B,CAACC,MAAAA,GAAAA;AACxC,IAAA,OAAOC,eAASC,CAAAA,IAAI,CACjB,gBAAgBC,sBAAAA,GAAAA;;AAEf,QAAA,MAAMC,kBAAkBC,iBAAM,CAAA;AAC5BL,YAAAA,MAAAA,CAAOM,EAAE,CAACC,YAAY,CAAC,sBAAsBC,MAAM,EAAA;YACnD,CAACC,IAAAA,GAASC,OAAI,OAASC,EAAAA,IAAAA,CAAKC,KAAK,CAACH,IAAAA,CAAKI,KAAK,CAAGJ,EAAAA,IAAAA,CAAAA;YAC/CK,qBAAsB,CAAA,YAAA;AACvB,SAAA,CAAA;;AAGD,QAAA,MAAMC,iBAAiBV,iBAAM,CAAA;AAC3BL,YAAAA,MAAAA,CAAOM,EAAE,CAACC,YAAY,CAAC,mBAAmBC,MAAM,EAAA;YAChDM,qBAAsB,CAAA,SAAA;AACvB,SAAA,CAAA;AAED,QAAA,MAAME,OAAU,GAAA;AAACZ,YAAAA,eAAAA;AAAiBW,YAAAA;AAAe,SAAA;QAEjD,KAAK,MAAMP,UAAUQ,OAAS,CAAA;YAC5B,WAAW,MAAMC,QAAQT,MAAQ,CAAA;gBAC/B,MAAMS,IAAAA;AACR;AACF;AACF,KAAA,EAAA,CAAA;AAEJ;AAEA,MAAMH,qBAAwB,GAAA,CAACI,IAAmC,GAAA,CAACL,SAAoB;AACrFK,YAAAA,IAAAA;AACAL,YAAAA;SACF,CAAA;;;;"}

View File

@@ -0,0 +1,37 @@
import { Readable } from 'stream';
import { chain } from 'stream-chain';
import { set } from 'lodash/fp';
/**
* Create a readable stream that export the Strapi app configuration
*/ const createConfigurationStream = (strapi)=>{
return Readable.from(async function* configurationGenerator() {
// Core Store
const coreStoreStream = chain([
strapi.db.queryBuilder('strapi::core-store').stream(),
(data)=>set('value', JSON.parse(data.value), data),
wrapConfigurationItem('core-store')
]);
// Webhook
const webhooksStream = chain([
strapi.db.queryBuilder('strapi::webhook').stream(),
wrapConfigurationItem('webhook')
]);
const streams = [
coreStoreStream,
webhooksStream
];
for (const stream of streams){
for await (const item of stream){
yield item;
}
}
}());
};
const wrapConfigurationItem = (type)=>(value)=>({
type,
value
});
export { createConfigurationStream };
//# sourceMappingURL=configuration.mjs.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"configuration.mjs","sources":["../../../../src/strapi/providers/local-source/configuration.ts"],"sourcesContent":["import { Readable } from 'stream';\nimport { chain } from 'stream-chain';\nimport { set } from 'lodash/fp';\nimport type { Core } from '@strapi/types';\n\nimport type { IConfiguration } from '../../../../types';\n\n/**\n * Create a readable stream that export the Strapi app configuration\n */\nexport const createConfigurationStream = (strapi: Core.Strapi): Readable => {\n return Readable.from(\n (async function* configurationGenerator(): AsyncGenerator<IConfiguration> {\n // Core Store\n const coreStoreStream = chain([\n strapi.db.queryBuilder('strapi::core-store').stream(),\n (data) => set('value', JSON.parse(data.value), data),\n wrapConfigurationItem('core-store'),\n ]);\n\n // Webhook\n const webhooksStream = chain([\n strapi.db.queryBuilder('strapi::webhook').stream(),\n wrapConfigurationItem('webhook'),\n ]);\n\n const streams = [coreStoreStream, webhooksStream];\n\n for (const stream of streams) {\n for await (const item of stream) {\n yield item;\n }\n }\n })()\n );\n};\n\nconst wrapConfigurationItem = (type: 'core-store' | 'webhook') => (value: unknown) => ({\n type,\n value,\n});\n"],"names":["createConfigurationStream","strapi","Readable","from","configurationGenerator","coreStoreStream","chain","db","queryBuilder","stream","data","set","JSON","parse","value","wrapConfigurationItem","webhooksStream","streams","item","type"],"mappings":";;;;AAOA;;IAGaA,MAAAA,yBAAAA,GAA4B,CAACC,MAAAA,GAAAA;AACxC,IAAA,OAAOC,QAASC,CAAAA,IAAI,CACjB,gBAAgBC,sBAAAA,GAAAA;;AAEf,QAAA,MAAMC,kBAAkBC,KAAM,CAAA;AAC5BL,YAAAA,MAAAA,CAAOM,EAAE,CAACC,YAAY,CAAC,sBAAsBC,MAAM,EAAA;YACnD,CAACC,IAAAA,GAASC,IAAI,OAASC,EAAAA,IAAAA,CAAKC,KAAK,CAACH,IAAAA,CAAKI,KAAK,CAAGJ,EAAAA,IAAAA,CAAAA;YAC/CK,qBAAsB,CAAA,YAAA;AACvB,SAAA,CAAA;;AAGD,QAAA,MAAMC,iBAAiBV,KAAM,CAAA;AAC3BL,YAAAA,MAAAA,CAAOM,EAAE,CAACC,YAAY,CAAC,mBAAmBC,MAAM,EAAA;YAChDM,qBAAsB,CAAA,SAAA;AACvB,SAAA,CAAA;AAED,QAAA,MAAME,OAAU,GAAA;AAACZ,YAAAA,eAAAA;AAAiBW,YAAAA;AAAe,SAAA;QAEjD,KAAK,MAAMP,UAAUQ,OAAS,CAAA;YAC5B,WAAW,MAAMC,QAAQT,MAAQ,CAAA;gBAC/B,MAAMS,IAAAA;AACR;AACF;AACF,KAAA,EAAA,CAAA;AAEJ;AAEA,MAAMH,qBAAwB,GAAA,CAACI,IAAmC,GAAA,CAACL,SAAoB;AACrFK,YAAAA,IAAAA;AACAL,YAAAA;SACF,CAAA;;;;"}

View File

@@ -0,0 +1,13 @@
/// <reference types="node" />
import { Readable, Transform } from 'stream';
import type { Core } from '@strapi/types';
/**
* Generate and consume content-types streams in order to stream each entity individually
*/
export declare const createEntitiesStream: (strapi: Core.Strapi) => Readable;
/**
* Create an entity transform stream which convert the output of
* the multi-content-types stream to the transfer entity format
*/
export declare const createEntitiesTransformStream: () => Transform;
//# sourceMappingURL=entities.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"entities.d.ts","sourceRoot":"","sources":["../../../../src/strapi/providers/local-source/entities.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC7C,OAAO,KAAK,EAAE,IAAI,EAAU,MAAM,eAAe,CAAC;AAKlD;;GAEG;AACH,eAAO,MAAM,oBAAoB,WAAY,KAAK,MAAM,KAAG,QAuC1D,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,6BAA6B,QAAO,SAchD,CAAC"}

View File

@@ -0,0 +1,62 @@
'use strict';
var stream = require('stream');
var entity = require('../../queries/entity.js');
require('lodash/fp');
/**
* Generate and consume content-types streams in order to stream each entity individually
*/ const createEntitiesStream = (strapi)=>{
const contentTypes = Object.values(strapi.contentTypes);
async function* contentTypeStreamGenerator() {
for (const contentType of contentTypes){
const query = entity.createEntityQuery(strapi).call(null, contentType.uid);
const stream = strapi.db// Create a query builder instance (default type is 'select')
.queryBuilder(contentType.uid)// Fetch all columns
.select('*')// Apply the populate
.populate(query.deepPopulateComponentLikeQuery)// Get a readable stream
.stream();
yield {
contentType,
stream
};
}
}
return stream.Readable.from(async function* entitiesGenerator() {
for await (const { stream, contentType } of contentTypeStreamGenerator()){
try {
for await (const entity of stream){
yield {
entity,
contentType
};
}
} catch {
// ignore
} finally{
stream.destroy();
}
}
}());
};
/**
* Create an entity transform stream which convert the output of
* the multi-content-types stream to the transfer entity format
*/ const createEntitiesTransformStream = ()=>{
return new stream.Transform({
objectMode: true,
transform (data, _encoding, callback) {
const { entity, contentType } = data;
const { id, ...attributes } = entity;
callback(null, {
type: contentType.uid,
id,
data: attributes
});
}
});
};
exports.createEntitiesStream = createEntitiesStream;
exports.createEntitiesTransformStream = createEntitiesTransformStream;
//# sourceMappingURL=entities.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"entities.js","sources":["../../../../src/strapi/providers/local-source/entities.ts"],"sourcesContent":["import { Readable, Transform } from 'stream';\nimport type { Core, Struct } from '@strapi/types';\n\nimport * as shared from '../../queries';\nimport { IEntity } from '../../../../types';\n\n/**\n * Generate and consume content-types streams in order to stream each entity individually\n */\nexport const createEntitiesStream = (strapi: Core.Strapi): Readable => {\n const contentTypes: Struct.ContentTypeSchema[] = Object.values(strapi.contentTypes);\n\n async function* contentTypeStreamGenerator() {\n for (const contentType of contentTypes) {\n const query = shared.entity.createEntityQuery(strapi).call(null, contentType.uid);\n\n const stream: Readable = strapi.db\n // Create a query builder instance (default type is 'select')\n .queryBuilder(contentType.uid)\n // Fetch all columns\n .select('*')\n // Apply the populate\n .populate(query.deepPopulateComponentLikeQuery)\n // Get a readable stream\n .stream();\n\n yield { contentType, stream };\n }\n }\n\n return Readable.from(\n (async function* entitiesGenerator(): AsyncGenerator<{\n entity: IEntity;\n contentType: Struct.ContentTypeSchema;\n }> {\n for await (const { stream, contentType } of contentTypeStreamGenerator()) {\n try {\n for await (const entity of stream) {\n yield { entity, contentType };\n }\n } catch {\n // ignore\n } finally {\n stream.destroy();\n }\n }\n })()\n );\n};\n\n/**\n * Create an entity transform stream which convert the output of\n * the multi-content-types stream to the transfer entity format\n */\nexport const createEntitiesTransformStream = (): Transform => {\n return new Transform({\n objectMode: true,\n transform(data, _encoding, callback) {\n const { entity, contentType } = data;\n const { id, ...attributes } = entity;\n\n callback(null, {\n type: contentType.uid,\n id,\n data: attributes,\n });\n },\n });\n};\n"],"names":["createEntitiesStream","strapi","contentTypes","Object","values","contentTypeStreamGenerator","contentType","query","shared","call","uid","stream","db","queryBuilder","select","populate","deepPopulateComponentLikeQuery","Readable","from","entitiesGenerator","entity","destroy","createEntitiesTransformStream","Transform","objectMode","transform","data","_encoding","callback","id","attributes","type"],"mappings":";;;;;;AAMA;;IAGaA,MAAAA,oBAAAA,GAAuB,CAACC,MAAAA,GAAAA;AACnC,IAAA,MAAMC,YAA2CC,GAAAA,MAAAA,CAAOC,MAAM,CAACH,OAAOC,YAAY,CAAA;IAElF,gBAAgBG,0BAAAA,GAAAA;QACd,KAAK,MAAMC,eAAeJ,YAAc,CAAA;YACtC,MAAMK,KAAAA,GAAQC,wBAA+B,CAACP,MAAAA,CAAAA,CAAQQ,IAAI,CAAC,IAAMH,EAAAA,WAAAA,CAAYI,GAAG,CAAA;AAEhF,YAAA,MAAMC,MAAmBV,GAAAA,MAAAA,CAAOW,EAC9B;AACCC,aAAAA,YAAY,CAACP,WAAAA,CAAYI,GAAG,CAC7B;aACCI,MAAM,CAAC,IACR;AACCC,aAAAA,QAAQ,CAACR,KAAAA,CAAMS,8BAA8B,CAC9C;aACCL,MAAM,EAAA;YAET,MAAM;AAAEL,gBAAAA,WAAAA;AAAaK,gBAAAA;AAAO,aAAA;AAC9B;AACF;AAEA,IAAA,OAAOM,eAASC,CAAAA,IAAI,CACjB,gBAAgBC,iBAAAA,GAAAA;AAIf,QAAA,WAAW,MAAM,EAAER,MAAM,EAAEL,WAAW,EAAE,IAAID,0BAA8B,EAAA,CAAA;YACxE,IAAI;gBACF,WAAW,MAAMe,UAAUT,MAAQ,CAAA;oBACjC,MAAM;AAAES,wBAAAA,MAAAA;AAAQd,wBAAAA;AAAY,qBAAA;AAC9B;AACF,aAAA,CAAE,OAAM;;aAEE,QAAA;AACRK,gBAAAA,MAAAA,CAAOU,OAAO,EAAA;AAChB;AACF;AACF,KAAA,EAAA,CAAA;AAEJ;AAEA;;;UAIaC,6BAAgC,GAAA,IAAA;AAC3C,IAAA,OAAO,IAAIC,gBAAU,CAAA;QACnBC,UAAY,EAAA,IAAA;AACZC,QAAAA,SAAAA,CAAAA,CAAUC,IAAI,EAAEC,SAAS,EAAEC,QAAQ,EAAA;AACjC,YAAA,MAAM,EAAER,MAAM,EAAEd,WAAW,EAAE,GAAGoB,IAAAA;AAChC,YAAA,MAAM,EAAEG,EAAE,EAAE,GAAGC,YAAY,GAAGV,MAAAA;AAE9BQ,YAAAA,QAAAA,CAAS,IAAM,EAAA;AACbG,gBAAAA,IAAAA,EAAMzB,YAAYI,GAAG;AACrBmB,gBAAAA,EAAAA;gBACAH,IAAMI,EAAAA;AACR,aAAA,CAAA;AACF;AACF,KAAA,CAAA;AACF;;;;;"}

View File

@@ -0,0 +1,59 @@
import { Readable, Transform } from 'stream';
import { createEntityQuery } from '../../queries/entity.mjs';
import 'lodash/fp';
/**
* Generate and consume content-types streams in order to stream each entity individually
*/ const createEntitiesStream = (strapi)=>{
const contentTypes = Object.values(strapi.contentTypes);
async function* contentTypeStreamGenerator() {
for (const contentType of contentTypes){
const query = createEntityQuery(strapi).call(null, contentType.uid);
const stream = strapi.db// Create a query builder instance (default type is 'select')
.queryBuilder(contentType.uid)// Fetch all columns
.select('*')// Apply the populate
.populate(query.deepPopulateComponentLikeQuery)// Get a readable stream
.stream();
yield {
contentType,
stream
};
}
}
return Readable.from(async function* entitiesGenerator() {
for await (const { stream, contentType } of contentTypeStreamGenerator()){
try {
for await (const entity of stream){
yield {
entity,
contentType
};
}
} catch {
// ignore
} finally{
stream.destroy();
}
}
}());
};
/**
* Create an entity transform stream which convert the output of
* the multi-content-types stream to the transfer entity format
*/ const createEntitiesTransformStream = ()=>{
return new Transform({
objectMode: true,
transform (data, _encoding, callback) {
const { entity, contentType } = data;
const { id, ...attributes } = entity;
callback(null, {
type: contentType.uid,
id,
data: attributes
});
}
});
};
export { createEntitiesStream, createEntitiesTransformStream };
//# sourceMappingURL=entities.mjs.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"entities.mjs","sources":["../../../../src/strapi/providers/local-source/entities.ts"],"sourcesContent":["import { Readable, Transform } from 'stream';\nimport type { Core, Struct } from '@strapi/types';\n\nimport * as shared from '../../queries';\nimport { IEntity } from '../../../../types';\n\n/**\n * Generate and consume content-types streams in order to stream each entity individually\n */\nexport const createEntitiesStream = (strapi: Core.Strapi): Readable => {\n const contentTypes: Struct.ContentTypeSchema[] = Object.values(strapi.contentTypes);\n\n async function* contentTypeStreamGenerator() {\n for (const contentType of contentTypes) {\n const query = shared.entity.createEntityQuery(strapi).call(null, contentType.uid);\n\n const stream: Readable = strapi.db\n // Create a query builder instance (default type is 'select')\n .queryBuilder(contentType.uid)\n // Fetch all columns\n .select('*')\n // Apply the populate\n .populate(query.deepPopulateComponentLikeQuery)\n // Get a readable stream\n .stream();\n\n yield { contentType, stream };\n }\n }\n\n return Readable.from(\n (async function* entitiesGenerator(): AsyncGenerator<{\n entity: IEntity;\n contentType: Struct.ContentTypeSchema;\n }> {\n for await (const { stream, contentType } of contentTypeStreamGenerator()) {\n try {\n for await (const entity of stream) {\n yield { entity, contentType };\n }\n } catch {\n // ignore\n } finally {\n stream.destroy();\n }\n }\n })()\n );\n};\n\n/**\n * Create an entity transform stream which convert the output of\n * the multi-content-types stream to the transfer entity format\n */\nexport const createEntitiesTransformStream = (): Transform => {\n return new Transform({\n objectMode: true,\n transform(data, _encoding, callback) {\n const { entity, contentType } = data;\n const { id, ...attributes } = entity;\n\n callback(null, {\n type: contentType.uid,\n id,\n data: attributes,\n });\n },\n });\n};\n"],"names":["createEntitiesStream","strapi","contentTypes","Object","values","contentTypeStreamGenerator","contentType","query","shared","call","uid","stream","db","queryBuilder","select","populate","deepPopulateComponentLikeQuery","Readable","from","entitiesGenerator","entity","destroy","createEntitiesTransformStream","Transform","objectMode","transform","data","_encoding","callback","id","attributes","type"],"mappings":";;;;AAMA;;IAGaA,MAAAA,oBAAAA,GAAuB,CAACC,MAAAA,GAAAA;AACnC,IAAA,MAAMC,YAA2CC,GAAAA,MAAAA,CAAOC,MAAM,CAACH,OAAOC,YAAY,CAAA;IAElF,gBAAgBG,0BAAAA,GAAAA;QACd,KAAK,MAAMC,eAAeJ,YAAc,CAAA;YACtC,MAAMK,KAAAA,GAAQC,iBAA+B,CAACP,MAAAA,CAAAA,CAAQQ,IAAI,CAAC,IAAMH,EAAAA,WAAAA,CAAYI,GAAG,CAAA;AAEhF,YAAA,MAAMC,MAAmBV,GAAAA,MAAAA,CAAOW,EAC9B;AACCC,aAAAA,YAAY,CAACP,WAAAA,CAAYI,GAAG,CAC7B;aACCI,MAAM,CAAC,IACR;AACCC,aAAAA,QAAQ,CAACR,KAAAA,CAAMS,8BAA8B,CAC9C;aACCL,MAAM,EAAA;YAET,MAAM;AAAEL,gBAAAA,WAAAA;AAAaK,gBAAAA;AAAO,aAAA;AAC9B;AACF;AAEA,IAAA,OAAOM,QAASC,CAAAA,IAAI,CACjB,gBAAgBC,iBAAAA,GAAAA;AAIf,QAAA,WAAW,MAAM,EAAER,MAAM,EAAEL,WAAW,EAAE,IAAID,0BAA8B,EAAA,CAAA;YACxE,IAAI;gBACF,WAAW,MAAMe,UAAUT,MAAQ,CAAA;oBACjC,MAAM;AAAES,wBAAAA,MAAAA;AAAQd,wBAAAA;AAAY,qBAAA;AAC9B;AACF,aAAA,CAAE,OAAM;;aAEE,QAAA;AACRK,gBAAAA,MAAAA,CAAOU,OAAO,EAAA;AAChB;AACF;AACF,KAAA,EAAA,CAAA;AAEJ;AAEA;;;UAIaC,6BAAgC,GAAA,IAAA;AAC3C,IAAA,OAAO,IAAIC,SAAU,CAAA;QACnBC,UAAY,EAAA,IAAA;AACZC,QAAAA,SAAAA,CAAAA,CAAUC,IAAI,EAAEC,SAAS,EAAEC,QAAQ,EAAA;AACjC,YAAA,MAAM,EAAER,MAAM,EAAEd,WAAW,EAAE,GAAGoB,IAAAA;AAChC,YAAA,MAAM,EAAEG,EAAE,EAAE,GAAGC,YAAY,GAAGV,MAAAA;AAE9BQ,YAAAA,QAAAA,CAAS,IAAM,EAAA;AACbG,gBAAAA,IAAAA,EAAMzB,YAAYI,GAAG;AACrBmB,gBAAAA,EAAAA;gBACAH,IAAMI,EAAAA;AACR,aAAA,CAAA;AACF;AACF,KAAA,CAAA;AACF;;;;"}

View File

@@ -0,0 +1,30 @@
/// <reference types="node" />
import { Readable } from 'stream';
import type { Core, Struct } from '@strapi/types';
import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types';
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
export interface ILocalStrapiSourceProviderOptions {
getStrapi(): Core.Strapi | Promise<Core.Strapi>;
autoDestroy?: boolean;
}
export declare const createLocalStrapiSourceProvider: (options: ILocalStrapiSourceProviderOptions) => LocalStrapiSourceProvider;
declare class LocalStrapiSourceProvider implements ISourceProvider {
#private;
name: string;
type: ProviderType;
options: ILocalStrapiSourceProviderOptions;
strapi?: Core.Strapi;
constructor(options: ILocalStrapiSourceProviderOptions);
bootstrap(diagnostics?: IDiagnosticReporter): Promise<void>;
close(): Promise<void>;
getMetadata(): IMetadata;
createEntitiesReadStream(): Promise<Readable>;
createLinksReadStream(): Readable;
createConfigurationReadStream(): Readable;
getSchemas(): Record<string, Struct.Schema>;
createSchemasReadStream(): Readable;
createAssetsReadStream(): Readable;
}
export type ILocalStrapiSourceProvider = InstanceType<typeof LocalStrapiSourceProvider>;
export {};
//# sourceMappingURL=index.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/strapi/providers/local-source/index.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAClF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAQrE,MAAM,WAAW,iCAAiC;IAChD,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhD,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,eAAO,MAAM,+BAA+B,YAAa,iCAAiC,8BAEzF,CAAC;AAEF,cAAM,yBAA0B,YAAW,eAAe;;IACxD,IAAI,SAA0B;IAE9B,IAAI,EAAE,YAAY,CAAY;IAE9B,OAAO,EAAE,iCAAiC,CAAC;IAE3C,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAIT,OAAO,EAAE,iCAAiC;IAIhD,SAAS,CAAC,WAAW,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAiD3D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAU5B,WAAW,IAAI,SAAS;IAalB,wBAAwB,IAAI,OAAO,CAAC,QAAQ,CAAC;IAYnD,qBAAqB,IAAI,QAAQ;IAOjC,6BAA6B,IAAI,QAAQ;IAMzC,UAAU,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;IAW3C,uBAAuB,IAAI,QAAQ;IAInC,sBAAsB,IAAI,QAAQ;CAWnC;AAED,MAAM,MAAM,0BAA0B,GAAG,YAAY,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

View File

@@ -0,0 +1,154 @@
'use strict';
var stream = require('stream');
var streamChain = require('stream-chain');
var entities = require('./entities.js');
var links = require('./links.js');
var configuration = require('./configuration.js');
var assets = require('./assets.js');
require('crypto');
require('lodash/fp');
var schema = require('../../../utils/schema.js');
require('events');
var providers = require('../../../utils/providers.js');
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
const createLocalStrapiSourceProvider = (options)=>{
return new LocalStrapiSourceProvider(options);
};
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), /**
* Reports an error to the diagnostic reporter.
*/ _reportError = /*#__PURE__*/ _class_private_field_loose_key("_reportError"), /**
* Handles errors that occur in read streams.
*/ _handleStreamError = /*#__PURE__*/ _class_private_field_loose_key("_handleStreamError");
class LocalStrapiSourceProvider {
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
this.strapi = await this.options.getStrapi();
this.strapi.db.lifecycles.disable();
}
async close() {
const { autoDestroy } = this.options;
providers.assertValidStrapi(this.strapi);
this.strapi.db.lifecycles.enable();
// Basically `!== false` but more deterministic
if (autoDestroy === undefined || autoDestroy === true) {
await this.strapi?.destroy();
}
}
getMetadata() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting metadata');
const strapiVersion = strapi.config.get('info.strapi');
const createdAt = new Date().toISOString();
return {
createdAt,
strapi: {
version: strapiVersion
}
};
}
async createEntitiesReadStream() {
providers.assertValidStrapi(this.strapi, 'Not able to stream entities');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating entities read stream');
return streamChain.chain([
// Entities stream
entities.createEntitiesStream(this.strapi),
// Transform stream
entities.createEntitiesTransformStream()
]);
}
createLinksReadStream() {
providers.assertValidStrapi(this.strapi, 'Not able to stream links');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating links read stream');
return links.createLinksStream(this.strapi);
}
createConfigurationReadStream() {
providers.assertValidStrapi(this.strapi, 'Not able to stream configuration');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating configuration read stream');
return configuration.createConfigurationStream(this.strapi);
}
getSchemas() {
providers.assertValidStrapi(this.strapi, 'Not able to get Schemas');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting schemas');
const schemas = schema.schemasToValidJSON({
...this.strapi.contentTypes,
...this.strapi.components
});
return schema.mapSchemasValues(schemas);
}
createSchemasReadStream() {
return stream.Readable.from(Object.values(this.getSchemas()));
}
createAssetsReadStream() {
providers.assertValidStrapi(this.strapi, 'Not able to stream assets');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets read stream');
const stream = assets.createAssetsStream(this.strapi);
stream.on('error', (err)=>{
_class_private_field_loose_base(this, _handleStreamError)[_handleStreamError]('assets', err);
});
return stream;
}
constructor(options){
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _reportError, {
value: reportError
});
Object.defineProperty(this, _handleStreamError, {
value: handleStreamError
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
this.name = 'source::local-strapi';
this.type = 'source';
this.options = options;
}
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'local-source-provider'
},
kind: 'info'
});
}
function reportError(message, error) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
error,
severity: 'fatal',
name: error.name
},
kind: 'error'
});
}
function handleStreamError(streamType, err) {
const { message, stack } = err;
const errorMessage = `[Data transfer] Error in ${streamType} read stream: ${message}`;
const formattedError = {
message: errorMessage,
stack,
timestamp: new Date().toISOString()
};
this.strapi?.log.error(formattedError);
_class_private_field_loose_base(this, _reportError)[_reportError](formattedError.message, err);
}
exports.createLocalStrapiSourceProvider = createLocalStrapiSourceProvider;
//# sourceMappingURL=index.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,152 @@
import { Readable } from 'stream';
import { chain } from 'stream-chain';
import { createEntitiesStream, createEntitiesTransformStream } from './entities.mjs';
import { createLinksStream } from './links.mjs';
import { createConfigurationStream } from './configuration.mjs';
import { createAssetsStream } from './assets.mjs';
import 'crypto';
import 'lodash/fp';
import { schemasToValidJSON, mapSchemasValues } from '../../../utils/schema.mjs';
import 'events';
import { assertValidStrapi } from '../../../utils/providers.mjs';
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
const createLocalStrapiSourceProvider = (options)=>{
return new LocalStrapiSourceProvider(options);
};
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), /**
* Reports an error to the diagnostic reporter.
*/ _reportError = /*#__PURE__*/ _class_private_field_loose_key("_reportError"), /**
* Handles errors that occur in read streams.
*/ _handleStreamError = /*#__PURE__*/ _class_private_field_loose_key("_handleStreamError");
class LocalStrapiSourceProvider {
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
this.strapi = await this.options.getStrapi();
this.strapi.db.lifecycles.disable();
}
async close() {
const { autoDestroy } = this.options;
assertValidStrapi(this.strapi);
this.strapi.db.lifecycles.enable();
// Basically `!== false` but more deterministic
if (autoDestroy === undefined || autoDestroy === true) {
await this.strapi?.destroy();
}
}
getMetadata() {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting metadata');
const strapiVersion = strapi.config.get('info.strapi');
const createdAt = new Date().toISOString();
return {
createdAt,
strapi: {
version: strapiVersion
}
};
}
async createEntitiesReadStream() {
assertValidStrapi(this.strapi, 'Not able to stream entities');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating entities read stream');
return chain([
// Entities stream
createEntitiesStream(this.strapi),
// Transform stream
createEntitiesTransformStream()
]);
}
createLinksReadStream() {
assertValidStrapi(this.strapi, 'Not able to stream links');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating links read stream');
return createLinksStream(this.strapi);
}
createConfigurationReadStream() {
assertValidStrapi(this.strapi, 'Not able to stream configuration');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating configuration read stream');
return createConfigurationStream(this.strapi);
}
getSchemas() {
assertValidStrapi(this.strapi, 'Not able to get Schemas');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('getting schemas');
const schemas = schemasToValidJSON({
...this.strapi.contentTypes,
...this.strapi.components
});
return mapSchemasValues(schemas);
}
createSchemasReadStream() {
return Readable.from(Object.values(this.getSchemas()));
}
createAssetsReadStream() {
assertValidStrapi(this.strapi, 'Not able to stream assets');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating assets read stream');
const stream = createAssetsStream(this.strapi);
stream.on('error', (err)=>{
_class_private_field_loose_base(this, _handleStreamError)[_handleStreamError]('assets', err);
});
return stream;
}
constructor(options){
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _reportError, {
value: reportError
});
Object.defineProperty(this, _handleStreamError, {
value: handleStreamError
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
this.name = 'source::local-strapi';
this.type = 'source';
this.options = options;
}
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'local-source-provider'
},
kind: 'info'
});
}
function reportError(message, error) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
error,
severity: 'fatal',
name: error.name
},
kind: 'error'
});
}
function handleStreamError(streamType, err) {
const { message, stack } = err;
const errorMessage = `[Data transfer] Error in ${streamType} read stream: ${message}`;
const formattedError = {
message: errorMessage,
stack,
timestamp: new Date().toISOString()
};
this.strapi?.log.error(formattedError);
_class_private_field_loose_base(this, _reportError)[_reportError](formattedError.message, err);
}
export { createLocalStrapiSourceProvider };
//# sourceMappingURL=index.mjs.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,8 @@
/// <reference types="node" />
import { Readable } from 'stream';
import type { Core } from '@strapi/types';
/**
* Create a Readable which will stream all the links from a Strapi instance
*/
export declare const createLinksStream: (strapi: Core.Strapi) => Readable;
//# sourceMappingURL=links.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"links.d.ts","sourceRoot":"","sources":["../../../../src/strapi/providers/local-source/links.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAK1C;;GAEG;AACH,eAAO,MAAM,iBAAiB,WAAY,KAAK,MAAM,KAAG,QAiBvD,CAAC"}

View File

@@ -0,0 +1,26 @@
'use strict';
var stream = require('stream');
var link = require('../../queries/link.js');
/**
* Create a Readable which will stream all the links from a Strapi instance
*/ const createLinksStream = (strapi)=>{
const uids = [
...Object.keys(strapi.contentTypes),
...Object.keys(strapi.components)
];
// Async generator stream that returns every link from a Strapi instance
return stream.Readable.from(async function* linkGenerator() {
const query = link.createLinkQuery(strapi);
for (const uid of uids){
const generator = query().generateAll(uid);
for await (const link of generator){
yield link;
}
}
}());
};
exports.createLinksStream = createLinksStream;
//# sourceMappingURL=links.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"links.js","sources":["../../../../src/strapi/providers/local-source/links.ts"],"sourcesContent":["import { Readable } from 'stream';\nimport type { Core } from '@strapi/types';\n\nimport type { ILink } from '../../../../types';\nimport { createLinkQuery } from '../../queries/link';\n\n/**\n * Create a Readable which will stream all the links from a Strapi instance\n */\nexport const createLinksStream = (strapi: Core.Strapi): Readable => {\n const uids = [...Object.keys(strapi.contentTypes), ...Object.keys(strapi.components)] as string[];\n\n // Async generator stream that returns every link from a Strapi instance\n return Readable.from(\n (async function* linkGenerator(): AsyncGenerator<ILink> {\n const query = createLinkQuery(strapi);\n\n for (const uid of uids) {\n const generator = query().generateAll(uid);\n\n for await (const link of generator) {\n yield link;\n }\n }\n })()\n );\n};\n"],"names":["createLinksStream","strapi","uids","Object","keys","contentTypes","components","Readable","from","linkGenerator","query","createLinkQuery","uid","generator","generateAll","link"],"mappings":";;;;;AAMA;;IAGaA,MAAAA,iBAAAA,GAAoB,CAACC,MAAAA,GAAAA;AAChC,IAAA,MAAMC,IAAO,GAAA;WAAIC,MAAOC,CAAAA,IAAI,CAACH,MAAAA,CAAOI,YAAY,CAAA;WAAMF,MAAOC,CAAAA,IAAI,CAACH,MAAAA,CAAOK,UAAU;AAAE,KAAA;;AAGrF,IAAA,OAAOC,eAASC,CAAAA,IAAI,CACjB,gBAAgBC,aAAAA,GAAAA;AACf,QAAA,MAAMC,QAAQC,oBAAgBV,CAAAA,MAAAA,CAAAA;QAE9B,KAAK,MAAMW,OAAOV,IAAM,CAAA;YACtB,MAAMW,SAAAA,GAAYH,KAAQI,EAAAA,CAAAA,WAAW,CAACF,GAAAA,CAAAA;YAEtC,WAAW,MAAMG,QAAQF,SAAW,CAAA;gBAClC,MAAME,IAAAA;AACR;AACF;AACF,KAAA,EAAA,CAAA;AAEJ;;;;"}

View File

@@ -0,0 +1,24 @@
import { Readable } from 'stream';
import { createLinkQuery } from '../../queries/link.mjs';
/**
* Create a Readable which will stream all the links from a Strapi instance
*/ const createLinksStream = (strapi)=>{
const uids = [
...Object.keys(strapi.contentTypes),
...Object.keys(strapi.components)
];
// Async generator stream that returns every link from a Strapi instance
return Readable.from(async function* linkGenerator() {
const query = createLinkQuery(strapi);
for (const uid of uids){
const generator = query().generateAll(uid);
for await (const link of generator){
yield link;
}
}
}());
};
export { createLinksStream };
//# sourceMappingURL=links.mjs.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"links.mjs","sources":["../../../../src/strapi/providers/local-source/links.ts"],"sourcesContent":["import { Readable } from 'stream';\nimport type { Core } from '@strapi/types';\n\nimport type { ILink } from '../../../../types';\nimport { createLinkQuery } from '../../queries/link';\n\n/**\n * Create a Readable which will stream all the links from a Strapi instance\n */\nexport const createLinksStream = (strapi: Core.Strapi): Readable => {\n const uids = [...Object.keys(strapi.contentTypes), ...Object.keys(strapi.components)] as string[];\n\n // Async generator stream that returns every link from a Strapi instance\n return Readable.from(\n (async function* linkGenerator(): AsyncGenerator<ILink> {\n const query = createLinkQuery(strapi);\n\n for (const uid of uids) {\n const generator = query().generateAll(uid);\n\n for await (const link of generator) {\n yield link;\n }\n }\n })()\n );\n};\n"],"names":["createLinksStream","strapi","uids","Object","keys","contentTypes","components","Readable","from","linkGenerator","query","createLinkQuery","uid","generator","generateAll","link"],"mappings":";;;AAMA;;IAGaA,MAAAA,iBAAAA,GAAoB,CAACC,MAAAA,GAAAA;AAChC,IAAA,MAAMC,IAAO,GAAA;WAAIC,MAAOC,CAAAA,IAAI,CAACH,MAAAA,CAAOI,YAAY,CAAA;WAAMF,MAAOC,CAAAA,IAAI,CAACH,MAAAA,CAAOK,UAAU;AAAE,KAAA;;AAGrF,IAAA,OAAOC,QAASC,CAAAA,IAAI,CACjB,gBAAgBC,aAAAA,GAAAA;AACf,QAAA,MAAMC,QAAQC,eAAgBV,CAAAA,MAAAA,CAAAA;QAE9B,KAAK,MAAMW,OAAOV,IAAM,CAAA;YACtB,MAAMW,SAAAA,GAAYH,KAAQI,EAAAA,CAAAA,WAAW,CAACF,GAAAA,CAAAA;YAEtC,WAAW,MAAMG,QAAQF,SAAW,CAAA;gBAClC,MAAME,IAAAA;AACR;AACF;AACF,KAAA,EAAA,CAAA;AAEJ;;;;"}

View File

@@ -0,0 +1,47 @@
/// <reference types="node" />
import { Writable } from 'stream';
import { WebSocket } from 'ws';
import type { Struct, Utils } from '@strapi/types';
import { createDispatcher } from '../utils';
import type { IDestinationProvider, IMetadata, ProviderType, TransferStage } from '../../../../types';
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
import type { Auth } from '../../../../types/remote/protocol';
import type { ILocalStrapiDestinationProviderOptions } from '../local-destination';
export interface IRemoteStrapiDestinationProviderOptions extends Pick<ILocalStrapiDestinationProviderOptions, 'restore' | 'strategy'> {
url: URL;
auth?: Auth.ITransferTokenAuth;
retryMessageOptions?: {
retryMessageTimeout: number;
retryMessageMaxRetries: number;
};
}
declare class RemoteStrapiDestinationProvider implements IDestinationProvider {
#private;
name: string;
type: ProviderType;
options: IRemoteStrapiDestinationProviderOptions;
ws: WebSocket | null;
dispatcher: ReturnType<typeof createDispatcher> | null;
transferID: string | null;
stats: {
[TStage in Exclude<TransferStage, 'schemas'>]: {
count: number;
};
};
constructor(options: IRemoteStrapiDestinationProviderOptions);
private resetStats;
initTransfer(): Promise<string>;
bootstrap(diagnostics?: IDiagnosticReporter): Promise<void>;
close(): Promise<void>;
getMetadata(): Promise<IMetadata | null> | null;
beforeTransfer(): Promise<void>;
rollback(): Promise<void>;
getSchemas(): Promise<Utils.String.Dict<Struct.Schema> | null>;
createEntitiesWriteStream(): Writable;
createLinksWriteStream(): Writable;
createConfigurationWriteStream(): Writable;
createAssetsWriteStream(): Writable | Promise<Writable>;
}
export declare const createRemoteStrapiDestinationProvider: (options: IRemoteStrapiDestinationProviderOptions) => RemoteStrapiDestinationProvider;
export {};
//# sourceMappingURL=index.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/strapi/providers/remote-destination/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAE/B,OAAO,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,EAAE,gBAAgB,EAAyC,MAAM,UAAU,CAAC;AAEnF,OAAO,KAAK,EACV,oBAAoB,EACpB,SAAS,EACT,YAAY,EAEZ,aAAa,EAEd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,KAAK,EAAkB,IAAI,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,KAAK,EAAE,sCAAsC,EAAE,MAAM,sBAAsB,CAAC;AAInF,MAAM,WAAW,uCACf,SAAQ,IAAI,CAAC,sCAAsC,EAAE,SAAS,GAAG,UAAU,CAAC;IAC5E,GAAG,EAAE,GAAG,CAAC;IACT,IAAI,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC;IAC/B,mBAAmB,CAAC,EAAE;QACpB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;CACH;AAID,cAAM,+BAAgC,YAAW,oBAAoB;;IACnE,IAAI,SAAgC;IAEpC,IAAI,EAAE,YAAY,CAAiB;IAEnC,OAAO,EAAE,uCAAuC,CAAC;IAEjD,EAAE,EAAE,SAAS,GAAG,IAAI,CAAC;IAErB,UAAU,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,GAAG,IAAI,CAAC;IAEvD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,KAAK,EAAG;SAAG,MAAM,IAAI,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,GAAG;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE;KAAE,CAAC;gBAIjE,OAAO,EAAE,uCAAuC;IAS5D,OAAO,CAAC,UAAU;IASZ,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAsK/B,SAAS,CAAC,WAAW,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IA+D3D,KAAK;IAuBX,WAAW;IAIL,cAAc;IAId,QAAQ;IAId,UAAU;IAQV,yBAAyB,IAAI,QAAQ;IAIrC,sBAAsB,IAAI,QAAQ;IAIlC,8BAA8B,IAAI,QAAQ;IAI1C,uBAAuB,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;CAiFxD;AAED,eAAO,MAAM,qCAAqC,YACvC,uCAAuC,oCAGjD,CAAC"}

View File

@@ -0,0 +1,392 @@
'use strict';
var crypto = require('crypto');
var stream = require('stream');
var fp = require('lodash/fp');
var utils = require('../utils.js');
var constants = require('../../remote/constants.js');
var providers = require('../../../errors/providers.js');
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
const jsonLength = (obj)=>Buffer.byteLength(JSON.stringify(obj));
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _startStepOnce = /*#__PURE__*/ _class_private_field_loose_key("_startStepOnce"), _startStep = /*#__PURE__*/ _class_private_field_loose_key("_startStep"), _endStep = /*#__PURE__*/ _class_private_field_loose_key("_endStep"), _streamStep = /*#__PURE__*/ _class_private_field_loose_key("_streamStep"), _writeStream = /*#__PURE__*/ _class_private_field_loose_key("_writeStream"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo");
class RemoteStrapiDestinationProvider {
resetStats() {
this.stats = {
assets: {
count: 0
},
entities: {
count: 0
},
links: {
count: 0
},
configuration: {
count: 0
}
};
}
async initTransfer() {
const { strategy, restore } = this.options;
const query = this.dispatcher?.dispatchCommand({
command: 'init',
params: {
options: {
strategy,
restore
},
transfer: 'push'
}
});
const res = await query;
if (!res?.transferID) {
throw new providers.ProviderTransferError('Init failed, invalid response from the server');
}
this.resetStats();
return res.transferID;
}
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
const { url, auth } = this.options;
const validProtocols = [
'https:',
'http:'
];
let ws;
if (!validProtocols.includes(url.protocol)) {
throw new providers.ProviderValidationError(`Invalid protocol "${url.protocol}"`, {
check: 'url',
details: {
protocol: url.protocol,
validProtocols
}
});
}
const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
const wsUrl = `${wsProtocol}//${url.host}${utils.trimTrailingSlash(url.pathname)}${constants.TRANSFER_PATH}/push`;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('establishing websocket connection');
// No auth defined, trying public access for transfer
if (!auth) {
ws = await utils.connectToWebsocket(wsUrl, undefined, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else if (auth.type === 'token') {
const headers = {
Authorization: `Bearer ${auth.token}`
};
ws = await utils.connectToWebsocket(wsUrl, {
headers
}, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else {
throw new providers.ProviderValidationError('Auth method not available', {
check: 'auth.type',
details: {
auth: auth.type
}
});
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('established websocket connection');
this.ws = ws;
const { retryMessageOptions } = this.options;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating dispatcher');
this.dispatcher = utils.createDispatcher(this.ws, retryMessageOptions, (message)=>_class_private_field_loose_base(this, _reportInfo)[_reportInfo](message));
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('created dispatcher');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('initialize transfer');
this.transferID = await this.initTransfer();
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`initialized transfer ${this.transferID}`);
this.dispatcher.setTransferProperties({
id: this.transferID,
kind: 'push'
});
await this.dispatcher.dispatchTransferAction('bootstrap');
}
async close() {
// Gracefully close the remote transfer process
if (this.transferID && this.dispatcher) {
await this.dispatcher.dispatchTransferAction('close');
await this.dispatcher.dispatchCommand({
command: 'end',
params: {
transferID: this.transferID
}
});
}
await new Promise((resolve)=>{
const { ws } = this;
if (!ws || ws.CLOSED) {
resolve();
return;
}
ws.on('close', ()=>resolve()).close();
});
}
getMetadata() {
return this.dispatcher?.dispatchTransferAction('getMetadata') ?? null;
}
async beforeTransfer() {
await this.dispatcher?.dispatchTransferAction('beforeTransfer');
}
async rollback() {
await this.dispatcher?.dispatchTransferAction('rollback');
}
getSchemas() {
if (!this.dispatcher) {
return Promise.resolve(null);
}
return this.dispatcher.dispatchTransferAction('getSchemas');
}
createEntitiesWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('entities');
}
createLinksWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('links');
}
createConfigurationWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('configuration');
}
createAssetsWriteStream() {
let batch = [];
let hasStarted = false;
const batchSize = 1024 * 1024; // 1MB;
const batchLength = ()=>{
return batch.reduce((acc, chunk)=>chunk.action === 'stream' ? acc + chunk.data.byteLength : acc, 0);
};
const startAssetsTransferOnce = _class_private_field_loose_base(this, _startStepOnce)[_startStepOnce]('assets');
const flush = async ()=>{
const streamError = await _class_private_field_loose_base(this, _streamStep)[_streamStep]('assets', batch);
batch = [];
return streamError;
};
const safePush = async (chunk)=>{
batch.push(chunk);
if (batchLength() >= batchSize) {
const streamError = await flush();
if (streamError) {
throw streamError;
}
}
};
return new stream.Writable({
objectMode: true,
final: async (callback)=>{
if (batch.length > 0) {
await flush();
}
if (hasStarted) {
const { error: endStepError } = await _class_private_field_loose_base(this, _endStep)[_endStep]('assets');
if (endStepError) {
return callback(endStepError);
}
}
return callback(null);
},
async write (asset, _encoding, callback) {
const startError = await startAssetsTransferOnce();
if (startError) {
return callback(startError);
}
hasStarted = true;
const assetID = crypto.randomUUID();
const { filename, filepath, stats, stream, metadata } = asset;
try {
await safePush({
action: 'start',
assetID,
data: {
filename,
filepath,
stats,
metadata
}
});
for await (const chunk of stream){
await safePush({
action: 'stream',
assetID,
data: chunk
});
}
await safePush({
action: 'end',
assetID
});
callback();
} catch (error) {
if (error instanceof Error) {
callback(error);
}
}
}
});
}
constructor(options){
Object.defineProperty(this, _startStepOnce, {
value: startStepOnce
});
Object.defineProperty(this, _startStep, {
value: startStep
});
Object.defineProperty(this, _endStep, {
value: endStep
});
Object.defineProperty(this, _streamStep, {
value: streamStep
});
Object.defineProperty(this, _writeStream, {
value: writeStream
});
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
this.name = 'destination::remote-strapi';
this.type = 'destination';
this.options = options;
this.ws = null;
this.dispatcher = null;
this.transferID = null;
this.resetStats();
}
}
function startStepOnce(stage) {
return fp.once(()=>_class_private_field_loose_base(this, _startStep)[_startStep](stage));
}
async function startStep(step) {
try {
await this.dispatcher?.dispatchTransferStep({
action: 'start',
step
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new providers.ProviderTransferError(e);
}
return new providers.ProviderTransferError('Unexpected error');
}
this.stats[step] = {
count: 0
};
return null;
}
async function endStep(step) {
try {
const res = await this.dispatcher?.dispatchTransferStep({
action: 'end',
step
});
return {
stats: res?.stats ?? null,
error: null
};
} catch (e) {
if (e instanceof Error) {
return {
stats: null,
error: e
};
}
if (typeof e === 'string') {
return {
stats: null,
error: new providers.ProviderTransferError(e)
};
}
return {
stats: null,
error: new providers.ProviderTransferError('Unexpected error')
};
}
}
async function streamStep(step, message) {
try {
if (step === 'assets') {
const assetMessage = message;
this.stats[step].count += assetMessage.filter((data)=>data.action === 'start').length;
} else {
this.stats[step].count += message.length;
}
await this.dispatcher?.dispatchTransferStep({
action: 'stream',
step,
data: message
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new providers.ProviderTransferError(e);
}
return new providers.ProviderTransferError('Unexpected error');
}
return null;
}
function writeStream(step) {
const batchSize = 1024 * 1024; // 1MB;
const startTransferOnce = _class_private_field_loose_base(this, _startStepOnce)[_startStepOnce](step);
let batch = [];
const batchLength = ()=>jsonLength(batch);
return new stream.Writable({
objectMode: true,
final: async (callback)=>{
if (batch.length > 0) {
const streamError = await _class_private_field_loose_base(this, _streamStep)[_streamStep](step, batch);
batch = [];
if (streamError) {
return callback(streamError);
}
}
const { error, stats } = await _class_private_field_loose_base(this, _endStep)[_endStep](step);
const { count } = this.stats[step];
if (stats && (stats.started !== count || stats.finished !== count)) {
callback(new Error(`Data missing: sent ${this.stats[step].count} ${step}, recieved ${stats.started} and saved ${stats.finished} ${step}`));
}
callback(error);
},
write: async (chunk, _encoding, callback)=>{
const startError = await startTransferOnce();
if (startError) {
return callback(startError);
}
batch.push(chunk);
if (batchLength() >= batchSize) {
const streamError = await _class_private_field_loose_base(this, _streamStep)[_streamStep](step, batch);
batch = [];
if (streamError) {
return callback(streamError);
}
}
callback();
}
});
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'remote-destination-provider'
},
kind: 'info'
});
}
const createRemoteStrapiDestinationProvider = (options)=>{
return new RemoteStrapiDestinationProvider(options);
};
exports.createRemoteStrapiDestinationProvider = createRemoteStrapiDestinationProvider;
//# sourceMappingURL=index.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,390 @@
import { randomUUID } from 'crypto';
import { Writable } from 'stream';
import { once } from 'lodash/fp';
import { trimTrailingSlash, connectToWebsocket, createDispatcher } from '../utils.mjs';
import { TRANSFER_PATH } from '../../remote/constants.mjs';
import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers.mjs';
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
const jsonLength = (obj)=>Buffer.byteLength(JSON.stringify(obj));
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _startStepOnce = /*#__PURE__*/ _class_private_field_loose_key("_startStepOnce"), _startStep = /*#__PURE__*/ _class_private_field_loose_key("_startStep"), _endStep = /*#__PURE__*/ _class_private_field_loose_key("_endStep"), _streamStep = /*#__PURE__*/ _class_private_field_loose_key("_streamStep"), _writeStream = /*#__PURE__*/ _class_private_field_loose_key("_writeStream"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo");
class RemoteStrapiDestinationProvider {
resetStats() {
this.stats = {
assets: {
count: 0
},
entities: {
count: 0
},
links: {
count: 0
},
configuration: {
count: 0
}
};
}
async initTransfer() {
const { strategy, restore } = this.options;
const query = this.dispatcher?.dispatchCommand({
command: 'init',
params: {
options: {
strategy,
restore
},
transfer: 'push'
}
});
const res = await query;
if (!res?.transferID) {
throw new ProviderTransferError('Init failed, invalid response from the server');
}
this.resetStats();
return res.transferID;
}
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
const { url, auth } = this.options;
const validProtocols = [
'https:',
'http:'
];
let ws;
if (!validProtocols.includes(url.protocol)) {
throw new ProviderValidationError(`Invalid protocol "${url.protocol}"`, {
check: 'url',
details: {
protocol: url.protocol,
validProtocols
}
});
}
const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
const wsUrl = `${wsProtocol}//${url.host}${trimTrailingSlash(url.pathname)}${TRANSFER_PATH}/push`;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('establishing websocket connection');
// No auth defined, trying public access for transfer
if (!auth) {
ws = await connectToWebsocket(wsUrl, undefined, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else if (auth.type === 'token') {
const headers = {
Authorization: `Bearer ${auth.token}`
};
ws = await connectToWebsocket(wsUrl, {
headers
}, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else {
throw new ProviderValidationError('Auth method not available', {
check: 'auth.type',
details: {
auth: auth.type
}
});
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('established websocket connection');
this.ws = ws;
const { retryMessageOptions } = this.options;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating dispatcher');
this.dispatcher = createDispatcher(this.ws, retryMessageOptions, (message)=>_class_private_field_loose_base(this, _reportInfo)[_reportInfo](message));
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('created dispatcher');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('initialize transfer');
this.transferID = await this.initTransfer();
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`initialized transfer ${this.transferID}`);
this.dispatcher.setTransferProperties({
id: this.transferID,
kind: 'push'
});
await this.dispatcher.dispatchTransferAction('bootstrap');
}
async close() {
// Gracefully close the remote transfer process
if (this.transferID && this.dispatcher) {
await this.dispatcher.dispatchTransferAction('close');
await this.dispatcher.dispatchCommand({
command: 'end',
params: {
transferID: this.transferID
}
});
}
await new Promise((resolve)=>{
const { ws } = this;
if (!ws || ws.CLOSED) {
resolve();
return;
}
ws.on('close', ()=>resolve()).close();
});
}
getMetadata() {
return this.dispatcher?.dispatchTransferAction('getMetadata') ?? null;
}
async beforeTransfer() {
await this.dispatcher?.dispatchTransferAction('beforeTransfer');
}
async rollback() {
await this.dispatcher?.dispatchTransferAction('rollback');
}
getSchemas() {
if (!this.dispatcher) {
return Promise.resolve(null);
}
return this.dispatcher.dispatchTransferAction('getSchemas');
}
createEntitiesWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('entities');
}
createLinksWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('links');
}
createConfigurationWriteStream() {
return _class_private_field_loose_base(this, _writeStream)[_writeStream]('configuration');
}
createAssetsWriteStream() {
let batch = [];
let hasStarted = false;
const batchSize = 1024 * 1024; // 1MB;
const batchLength = ()=>{
return batch.reduce((acc, chunk)=>chunk.action === 'stream' ? acc + chunk.data.byteLength : acc, 0);
};
const startAssetsTransferOnce = _class_private_field_loose_base(this, _startStepOnce)[_startStepOnce]('assets');
const flush = async ()=>{
const streamError = await _class_private_field_loose_base(this, _streamStep)[_streamStep]('assets', batch);
batch = [];
return streamError;
};
const safePush = async (chunk)=>{
batch.push(chunk);
if (batchLength() >= batchSize) {
const streamError = await flush();
if (streamError) {
throw streamError;
}
}
};
return new Writable({
objectMode: true,
final: async (callback)=>{
if (batch.length > 0) {
await flush();
}
if (hasStarted) {
const { error: endStepError } = await _class_private_field_loose_base(this, _endStep)[_endStep]('assets');
if (endStepError) {
return callback(endStepError);
}
}
return callback(null);
},
async write (asset, _encoding, callback) {
const startError = await startAssetsTransferOnce();
if (startError) {
return callback(startError);
}
hasStarted = true;
const assetID = randomUUID();
const { filename, filepath, stats, stream, metadata } = asset;
try {
await safePush({
action: 'start',
assetID,
data: {
filename,
filepath,
stats,
metadata
}
});
for await (const chunk of stream){
await safePush({
action: 'stream',
assetID,
data: chunk
});
}
await safePush({
action: 'end',
assetID
});
callback();
} catch (error) {
if (error instanceof Error) {
callback(error);
}
}
}
});
}
constructor(options){
Object.defineProperty(this, _startStepOnce, {
value: startStepOnce
});
Object.defineProperty(this, _startStep, {
value: startStep
});
Object.defineProperty(this, _endStep, {
value: endStep
});
Object.defineProperty(this, _streamStep, {
value: streamStep
});
Object.defineProperty(this, _writeStream, {
value: writeStream
});
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
this.name = 'destination::remote-strapi';
this.type = 'destination';
this.options = options;
this.ws = null;
this.dispatcher = null;
this.transferID = null;
this.resetStats();
}
}
function startStepOnce(stage) {
return once(()=>_class_private_field_loose_base(this, _startStep)[_startStep](stage));
}
async function startStep(step) {
try {
await this.dispatcher?.dispatchTransferStep({
action: 'start',
step
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new ProviderTransferError(e);
}
return new ProviderTransferError('Unexpected error');
}
this.stats[step] = {
count: 0
};
return null;
}
async function endStep(step) {
try {
const res = await this.dispatcher?.dispatchTransferStep({
action: 'end',
step
});
return {
stats: res?.stats ?? null,
error: null
};
} catch (e) {
if (e instanceof Error) {
return {
stats: null,
error: e
};
}
if (typeof e === 'string') {
return {
stats: null,
error: new ProviderTransferError(e)
};
}
return {
stats: null,
error: new ProviderTransferError('Unexpected error')
};
}
}
async function streamStep(step, message) {
try {
if (step === 'assets') {
const assetMessage = message;
this.stats[step].count += assetMessage.filter((data)=>data.action === 'start').length;
} else {
this.stats[step].count += message.length;
}
await this.dispatcher?.dispatchTransferStep({
action: 'stream',
step,
data: message
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new ProviderTransferError(e);
}
return new ProviderTransferError('Unexpected error');
}
return null;
}
function writeStream(step) {
const batchSize = 1024 * 1024; // 1MB;
const startTransferOnce = _class_private_field_loose_base(this, _startStepOnce)[_startStepOnce](step);
let batch = [];
const batchLength = ()=>jsonLength(batch);
return new Writable({
objectMode: true,
final: async (callback)=>{
if (batch.length > 0) {
const streamError = await _class_private_field_loose_base(this, _streamStep)[_streamStep](step, batch);
batch = [];
if (streamError) {
return callback(streamError);
}
}
const { error, stats } = await _class_private_field_loose_base(this, _endStep)[_endStep](step);
const { count } = this.stats[step];
if (stats && (stats.started !== count || stats.finished !== count)) {
callback(new Error(`Data missing: sent ${this.stats[step].count} ${step}, recieved ${stats.started} and saved ${stats.finished} ${step}`));
}
callback(error);
},
write: async (chunk, _encoding, callback)=>{
const startError = await startTransferOnce();
if (startError) {
return callback(startError);
}
batch.push(chunk);
if (batchLength() >= batchSize) {
const streamError = await _class_private_field_loose_base(this, _streamStep)[_streamStep](step, batch);
batch = [];
if (streamError) {
return callback(streamError);
}
}
callback();
}
});
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'remote-destination-provider'
},
kind: 'info'
});
}
const createRemoteStrapiDestinationProvider = (options)=>{
return new RemoteStrapiDestinationProvider(options);
};
export { createRemoteStrapiDestinationProvider };
//# sourceMappingURL=index.mjs.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,43 @@
/// <reference types="node" />
import { Readable, Writable } from 'stream';
import type { Struct, Utils } from '@strapi/types';
import { WebSocket } from 'ws';
import type { IMetadata, ISourceProvider, ISourceProviderTransferResults, MaybePromise, ProviderType } from '../../../../types';
import type { IDiagnosticReporter } from '../../../utils/diagnostic';
import { Auth } from '../../../../types/remote/protocol';
import { ILocalStrapiSourceProviderOptions } from '../local-source';
import { createDispatcher } from '../utils';
export interface IRemoteStrapiSourceProviderOptions extends ILocalStrapiSourceProviderOptions {
url: URL;
auth?: Auth.ITransferTokenAuth;
retryMessageOptions?: {
retryMessageTimeout: number;
retryMessageMaxRetries: number;
};
streamTimeout?: number;
}
declare class RemoteStrapiSourceProvider implements ISourceProvider {
#private;
name: string;
type: ProviderType;
options: IRemoteStrapiSourceProviderOptions;
ws: WebSocket | null;
dispatcher: ReturnType<typeof createDispatcher> | null;
defaultOptions: Partial<IRemoteStrapiSourceProviderOptions>;
constructor(options: IRemoteStrapiSourceProviderOptions);
results?: ISourceProviderTransferResults | undefined;
createEntitiesReadStream(): MaybePromise<Readable>;
createLinksReadStream(): MaybePromise<Readable>;
writeAsync: <T>(stream: Writable, data: T) => Promise<void>;
createAssetsReadStream(): Promise<Readable>;
createConfigurationReadStream(): MaybePromise<Readable>;
getMetadata(): Promise<IMetadata | null>;
assertValidProtocol(url: URL): void;
initTransfer(): Promise<string>;
bootstrap(diagnostics?: IDiagnosticReporter): Promise<void>;
close(): Promise<void>;
getSchemas(): Promise<Utils.String.Dict<Struct.Schema> | null>;
}
export declare const createRemoteStrapiSourceProvider: (options: IRemoteStrapiSourceProviderOptions) => RemoteStrapiSourceProvider;
export {};
//# sourceMappingURL=index.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/strapi/providers/remote-source/index.ts"],"names":[],"mappings":";AAAA,OAAO,EAAe,QAAQ,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AACzD,OAAO,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,OAAO,KAAK,EAEV,SAAS,EACT,eAAe,EACf,8BAA8B,EAC9B,YAAY,EAEZ,YAAY,EAEb,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,EAAkB,IAAI,EAAE,MAAM,mCAAmC,CAAC;AAGzE,OAAO,EAAE,iCAAiC,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAyC,MAAM,UAAU,CAAC;AAEnF,MAAM,WAAW,kCAAmC,SAAQ,iCAAiC;IAC3F,GAAG,EAAE,GAAG,CAAC;IACT,IAAI,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC;IAC/B,mBAAmB,CAAC,EAAE;QACpB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,sBAAsB,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAKD,cAAM,0BAA2B,YAAW,eAAe;;IACzD,IAAI,SAA2B;IAE/B,IAAI,EAAE,YAAY,CAAY;IAE9B,OAAO,EAAE,kCAAkC,CAAC;IAE5C,EAAE,EAAE,SAAS,GAAG,IAAI,CAAC;IAErB,UAAU,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,GAAG,IAAI,CAAC;IAEvD,cAAc,EAAE,OAAO,CAAC,kCAAkC,CAAC,CAEzD;gBAEU,OAAO,EAAE,kCAAkC;IAUvD,OAAO,CAAC,EAAE,8BAA8B,GAAG,SAAS,CAAC;IAuDrD,wBAAwB,IAAI,YAAY,CAAC,QAAQ,CAAC;IAIlD,qBAAqB,IAAI,YAAY,CAAC,QAAQ,CAAC;IAI/C,UAAU,cAAe,QAAQ,QAAQ,CAAC,mBAUxC;IAEI,sBAAsB,IAAI,OAAO,CAAC,QAAQ,CAAC;IAkMjD,6BAA6B,IAAI,YAAY,CAAC,QAAQ,CAAC;IAIjD,WAAW,IAAI,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAM9C,mBAAmB,CAAC,GAAG,EAAE,GAAG;IActB,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAyB/B,SAAS,CAAC,WAAW,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkD3D,KAAK;IAeL,UAAU;CAoDjB;AAED,eAAO,MAAM,gCAAgC,YAAa,kCAAkC,+BAE3F,CAAC"}

View File

@@ -0,0 +1,405 @@
'use strict';
var stream = require('stream');
var fp = require('lodash/fp');
var providers = require('../../../errors/providers.js');
var constants = require('../../remote/constants.js');
var utils = require('../utils.js');
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _createStageReadStream = /*#__PURE__*/ _class_private_field_loose_key("_createStageReadStream"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), _startStep = /*#__PURE__*/ _class_private_field_loose_key("_startStep"), _respond = /*#__PURE__*/ _class_private_field_loose_key("_respond"), _endStep = /*#__PURE__*/ _class_private_field_loose_key("_endStep");
class RemoteStrapiSourceProvider {
createEntitiesReadStream() {
return _class_private_field_loose_base(this, _createStageReadStream)[_createStageReadStream]('entities');
}
createLinksReadStream() {
return _class_private_field_loose_base(this, _createStageReadStream)[_createStageReadStream]('links');
}
async createAssetsReadStream() {
// Create the streams used to transfer the assets
const stream$1 = await _class_private_field_loose_base(this, _createStageReadStream)[_createStageReadStream]('assets');
const pass = new stream.PassThrough({
objectMode: true
});
// Init the asset map
const assets = {};
// Watch for stalled assets; if we don't receive a chunk within timeout, abort transfer
const resetTimeout = (assetID)=>{
if (assets[assetID].timeout) {
clearTimeout(assets[assetID].timeout);
}
assets[assetID].timeout = setTimeout(()=>{
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`Asset ${assetID} transfer stalled, aborting.`);
assets[assetID].status = 'errored';
assets[assetID].stream.destroy(new Error(`Asset ${assetID} transfer timed out`));
}, this.options.streamTimeout);
};
stream$1/**
* Process a payload of many transfer assets and performs the following tasks:
* - Start: creates a stream for new assets.
* - Stream: writes asset chunks to the asset's stream.
* - End: closes the stream after the asset s transferred and cleanup related resources.
*/ .on('data', async (payload)=>{
for (const item of payload){
const { action, assetID } = item;
// Creates the stream to send the incoming asset through
if (action === 'start') {
// if a transfer has already been started for the same asset ID, something is wrong
if (assets[assetID]) {
throw new Error(`Asset ${assetID} already started`);
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`Asset ${assetID} starting`);
// Register the asset
assets[assetID] = {
...item.data,
stream: new stream.PassThrough(),
status: 'ok',
queue: []
};
resetTimeout(assetID);
// Connect the individual asset stream to the main asset stage stream
// Note: nothing is transferred until data chunks are fed to the asset stream
await this.writeAsync(pass, assets[assetID]);
} else if (action === 'stream' || action === 'end') {
// If the asset hasn't been registered, or if it's been closed already, something is wrong
if (!assets[assetID]) {
throw new Error(`No id matching ${assetID} for stream action`);
}
// On every action, reset the timeout timer
if (action === 'stream') {
resetTimeout(assetID);
} else {
clearTimeout(assets[assetID].timeout);
}
if (assets[assetID].status === 'closed') {
throw new Error(`Asset ${assetID} is closed`);
}
assets[assetID].queue.push(item);
}
}
// each new payload will start new processQueue calls, which may cause some extra calls
// it's essentially saying "start processing this asset again, I added more data to the queue"
for(const assetID in assets){
if (Object.prototype.hasOwnProperty.call(assets, assetID)) {
const asset = assets[assetID];
if (asset.queue?.length > 0) {
await processQueue(assetID);
}
}
}
}).on('close', ()=>{
pass.end();
});
/**
* Start processing the queue for a given assetID
*
* Even though this is a loop that attempts to process the entire queue, it is safe to call this more than once
* for the same asset id because the queue is shared globally, the items are shifted off, and immediately written
*/ const processQueue = async (id)=>{
if (!assets[id]) {
throw new Error(`Failed to write asset chunk for "${id}". Asset not found.`);
}
const asset = assets[id];
const { status: currentStatus } = asset;
if ([
'closed',
'errored'
].includes(currentStatus)) {
throw new Error(`Failed to write asset chunk for "${id}". The asset is currently "${currentStatus}"`);
}
while(asset.queue.length > 0){
const data = asset.queue.shift();
if (!data) {
throw new Error(`Invalid chunk found for ${id}`);
}
try {
// if this is an end chunk, close the asset stream
if (data.action === 'end') {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`Ending asset stream for ${id}`);
await closeAssetStream(id);
break; // Exit the loop after closing the stream
}
// Save the current chunk
await writeChunkToStream(id, data);
} catch {
if (!assets[id]) {
throw new Error(`No id matching ${id} for writeAssetChunk`);
}
}
}
};
/**
* Writes a chunk of data to the asset's stream.
*
* Only check if the targeted asset exists, no other validation is done.
*/ const writeChunkToStream = async (id, data)=>{
const asset = assets[id];
if (!asset) {
throw new Error(`Failed to write asset chunk for "${id}". Asset not found.`);
}
const rawBuffer = data;
const chunk = Buffer.from(rawBuffer.data);
await this.writeAsync(asset.stream, chunk);
};
/**
* Closes the asset stream associated with the given ID.
*
* It deletes the stream for the asset upon successful closure.
*/ const closeAssetStream = async (id)=>{
if (!assets[id]) {
throw new Error(`Failed to close asset "${id}". Asset not found.`);
}
assets[id].status = 'closed';
await new Promise((resolve, reject)=>{
const { stream } = assets[id];
stream.on('close', ()=>{
resolve();
}).on('error', (e)=>{
assets[id].status = 'errored';
reject(new Error(`Failed to close asset "${id}". Asset stream error: ${e.toString()}`));
}).end();
});
};
return pass;
}
createConfigurationReadStream() {
return _class_private_field_loose_base(this, _createStageReadStream)[_createStageReadStream]('configuration');
}
async getMetadata() {
const metadata = await this.dispatcher?.dispatchTransferAction('getMetadata');
return metadata ?? null;
}
assertValidProtocol(url) {
const validProtocols = [
'https:',
'http:'
];
if (!validProtocols.includes(url.protocol)) {
throw new providers.ProviderValidationError(`Invalid protocol "${url.protocol}"`, {
check: 'url',
details: {
protocol: url.protocol,
validProtocols
}
});
}
}
async initTransfer() {
const query = this.dispatcher?.dispatchCommand({
command: 'init'
});
const res = await query;
if (!res?.transferID) {
throw new providers.ProviderTransferError('Init failed, invalid response from the server');
}
return res.transferID;
}
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
const { url, auth } = this.options;
let ws;
this.assertValidProtocol(url);
const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
const wsUrl = `${wsProtocol}//${url.host}${utils.trimTrailingSlash(url.pathname)}${constants.TRANSFER_PATH}/pull`;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('establishing websocket connection');
// No auth defined, trying public access for transfer
if (!auth) {
ws = await utils.connectToWebsocket(wsUrl, undefined, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else if (auth.type === 'token') {
const headers = {
Authorization: `Bearer ${auth.token}`
};
ws = await utils.connectToWebsocket(wsUrl, {
headers
}, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else {
throw new providers.ProviderValidationError('Auth method not available', {
check: 'auth.type',
details: {
auth: auth.type
}
});
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('established websocket connection');
this.ws = ws;
const { retryMessageOptions } = this.options;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating dispatcher');
this.dispatcher = utils.createDispatcher(this.ws, retryMessageOptions, (message)=>_class_private_field_loose_base(this, _reportInfo)[_reportInfo](message));
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating dispatcher');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('initialize transfer');
const transferID = await this.initTransfer();
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`initialized transfer ${transferID}`);
this.dispatcher.setTransferProperties({
id: transferID,
kind: 'pull'
});
await this.dispatcher.dispatchTransferAction('bootstrap');
}
async close() {
await this.dispatcher?.dispatchTransferAction('close');
await new Promise((resolve)=>{
const { ws } = this;
if (!ws || ws.CLOSED) {
resolve();
return;
}
ws.on('close', ()=>resolve()).close();
});
}
async getSchemas() {
const schemas = await this.dispatcher?.dispatchTransferAction('getSchemas');
return schemas ?? null;
}
constructor(options){
Object.defineProperty(this, _createStageReadStream, {
value: createStageReadStream
});
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _startStep, {
value: startStep
});
Object.defineProperty(this, _respond, {
value: respond
});
Object.defineProperty(this, _endStep, {
value: endStep
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
this.name = 'source::remote-strapi';
this.type = 'source';
this.defaultOptions = {
streamTimeout: 15000
};
this.writeAsync = (stream, data)=>{
return new Promise((resolve, reject)=>{
stream.write(data, (error)=>{
if (error) {
reject(error);
}
resolve();
});
});
};
this.options = {
...this.defaultOptions,
...options
};
this.ws = null;
this.dispatcher = null;
}
}
async function createStageReadStream(stage) {
const startResult = await _class_private_field_loose_base(this, _startStep)[_startStep](stage);
if (startResult instanceof Error) {
throw startResult;
}
const { id: processID } = startResult;
const stream$1 = new stream.PassThrough({
objectMode: true
});
const listener = async (raw)=>{
const parsed = JSON.parse(raw.toString());
// If not a message related to our transfer process, ignore it
if (!parsed.uuid || parsed?.data?.type !== 'transfer' || parsed?.data?.id !== processID) {
this.ws?.once('message', listener);
return;
}
const { uuid, data: message } = parsed;
const { ended, error, data } = message;
if (error) {
await _class_private_field_loose_base(this, _respond)[_respond](uuid);
stream$1.destroy(error);
return;
}
if (ended) {
await _class_private_field_loose_base(this, _respond)[_respond](uuid);
await _class_private_field_loose_base(this, _endStep)[_endStep](stage);
stream$1.end();
return;
}
// if we get a single items instead of a batch
for (const item of fp.castArray(data)){
stream$1.push(item);
}
this.ws?.once('message', listener);
await _class_private_field_loose_base(this, _respond)[_respond](uuid);
};
this.ws?.once('message', listener);
return stream$1;
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'remote-source-provider'
},
kind: 'info'
});
}
async function startStep(step) {
try {
return await this.dispatcher?.dispatchTransferStep({
action: 'start',
step
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new providers.ProviderTransferError(e);
}
return new providers.ProviderTransferError('Unexpected error');
}
}
async function respond(uuid) {
return new Promise((resolve, reject)=>{
this.ws?.send(JSON.stringify({
uuid
}), (e)=>{
if (e) {
reject(e);
} else {
resolve(e);
}
});
});
}
async function endStep(step) {
try {
await this.dispatcher?.dispatchTransferStep({
action: 'end',
step
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new providers.ProviderTransferError(e);
}
return new providers.ProviderTransferError('Unexpected error');
}
return null;
}
const createRemoteStrapiSourceProvider = (options)=>{
return new RemoteStrapiSourceProvider(options);
};
exports.createRemoteStrapiSourceProvider = createRemoteStrapiSourceProvider;
//# sourceMappingURL=index.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,403 @@
import { PassThrough } from 'stream';
import { castArray } from 'lodash/fp';
import { ProviderValidationError, ProviderTransferError } from '../../../errors/providers.mjs';
import { TRANSFER_PATH } from '../../remote/constants.mjs';
import { trimTrailingSlash, connectToWebsocket, createDispatcher } from '../utils.mjs';
function _class_private_field_loose_base(receiver, privateKey) {
if (!Object.prototype.hasOwnProperty.call(receiver, privateKey)) {
throw new TypeError("attempted to use private field on non-instance");
}
return receiver;
}
var id = 0;
function _class_private_field_loose_key(name) {
return "__private_" + id++ + "_" + name;
}
var _diagnostics = /*#__PURE__*/ _class_private_field_loose_key("_diagnostics"), _createStageReadStream = /*#__PURE__*/ _class_private_field_loose_key("_createStageReadStream"), _reportInfo = /*#__PURE__*/ _class_private_field_loose_key("_reportInfo"), _startStep = /*#__PURE__*/ _class_private_field_loose_key("_startStep"), _respond = /*#__PURE__*/ _class_private_field_loose_key("_respond"), _endStep = /*#__PURE__*/ _class_private_field_loose_key("_endStep");
class RemoteStrapiSourceProvider {
createEntitiesReadStream() {
return _class_private_field_loose_base(this, _createStageReadStream)[_createStageReadStream]('entities');
}
createLinksReadStream() {
return _class_private_field_loose_base(this, _createStageReadStream)[_createStageReadStream]('links');
}
async createAssetsReadStream() {
// Create the streams used to transfer the assets
const stream = await _class_private_field_loose_base(this, _createStageReadStream)[_createStageReadStream]('assets');
const pass = new PassThrough({
objectMode: true
});
// Init the asset map
const assets = {};
// Watch for stalled assets; if we don't receive a chunk within timeout, abort transfer
const resetTimeout = (assetID)=>{
if (assets[assetID].timeout) {
clearTimeout(assets[assetID].timeout);
}
assets[assetID].timeout = setTimeout(()=>{
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`Asset ${assetID} transfer stalled, aborting.`);
assets[assetID].status = 'errored';
assets[assetID].stream.destroy(new Error(`Asset ${assetID} transfer timed out`));
}, this.options.streamTimeout);
};
stream/**
* Process a payload of many transfer assets and performs the following tasks:
* - Start: creates a stream for new assets.
* - Stream: writes asset chunks to the asset's stream.
* - End: closes the stream after the asset s transferred and cleanup related resources.
*/ .on('data', async (payload)=>{
for (const item of payload){
const { action, assetID } = item;
// Creates the stream to send the incoming asset through
if (action === 'start') {
// if a transfer has already been started for the same asset ID, something is wrong
if (assets[assetID]) {
throw new Error(`Asset ${assetID} already started`);
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`Asset ${assetID} starting`);
// Register the asset
assets[assetID] = {
...item.data,
stream: new PassThrough(),
status: 'ok',
queue: []
};
resetTimeout(assetID);
// Connect the individual asset stream to the main asset stage stream
// Note: nothing is transferred until data chunks are fed to the asset stream
await this.writeAsync(pass, assets[assetID]);
} else if (action === 'stream' || action === 'end') {
// If the asset hasn't been registered, or if it's been closed already, something is wrong
if (!assets[assetID]) {
throw new Error(`No id matching ${assetID} for stream action`);
}
// On every action, reset the timeout timer
if (action === 'stream') {
resetTimeout(assetID);
} else {
clearTimeout(assets[assetID].timeout);
}
if (assets[assetID].status === 'closed') {
throw new Error(`Asset ${assetID} is closed`);
}
assets[assetID].queue.push(item);
}
}
// each new payload will start new processQueue calls, which may cause some extra calls
// it's essentially saying "start processing this asset again, I added more data to the queue"
for(const assetID in assets){
if (Object.prototype.hasOwnProperty.call(assets, assetID)) {
const asset = assets[assetID];
if (asset.queue?.length > 0) {
await processQueue(assetID);
}
}
}
}).on('close', ()=>{
pass.end();
});
/**
* Start processing the queue for a given assetID
*
* Even though this is a loop that attempts to process the entire queue, it is safe to call this more than once
* for the same asset id because the queue is shared globally, the items are shifted off, and immediately written
*/ const processQueue = async (id)=>{
if (!assets[id]) {
throw new Error(`Failed to write asset chunk for "${id}". Asset not found.`);
}
const asset = assets[id];
const { status: currentStatus } = asset;
if ([
'closed',
'errored'
].includes(currentStatus)) {
throw new Error(`Failed to write asset chunk for "${id}". The asset is currently "${currentStatus}"`);
}
while(asset.queue.length > 0){
const data = asset.queue.shift();
if (!data) {
throw new Error(`Invalid chunk found for ${id}`);
}
try {
// if this is an end chunk, close the asset stream
if (data.action === 'end') {
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`Ending asset stream for ${id}`);
await closeAssetStream(id);
break; // Exit the loop after closing the stream
}
// Save the current chunk
await writeChunkToStream(id, data);
} catch {
if (!assets[id]) {
throw new Error(`No id matching ${id} for writeAssetChunk`);
}
}
}
};
/**
* Writes a chunk of data to the asset's stream.
*
* Only check if the targeted asset exists, no other validation is done.
*/ const writeChunkToStream = async (id, data)=>{
const asset = assets[id];
if (!asset) {
throw new Error(`Failed to write asset chunk for "${id}". Asset not found.`);
}
const rawBuffer = data;
const chunk = Buffer.from(rawBuffer.data);
await this.writeAsync(asset.stream, chunk);
};
/**
* Closes the asset stream associated with the given ID.
*
* It deletes the stream for the asset upon successful closure.
*/ const closeAssetStream = async (id)=>{
if (!assets[id]) {
throw new Error(`Failed to close asset "${id}". Asset not found.`);
}
assets[id].status = 'closed';
await new Promise((resolve, reject)=>{
const { stream } = assets[id];
stream.on('close', ()=>{
resolve();
}).on('error', (e)=>{
assets[id].status = 'errored';
reject(new Error(`Failed to close asset "${id}". Asset stream error: ${e.toString()}`));
}).end();
});
};
return pass;
}
createConfigurationReadStream() {
return _class_private_field_loose_base(this, _createStageReadStream)[_createStageReadStream]('configuration');
}
async getMetadata() {
const metadata = await this.dispatcher?.dispatchTransferAction('getMetadata');
return metadata ?? null;
}
assertValidProtocol(url) {
const validProtocols = [
'https:',
'http:'
];
if (!validProtocols.includes(url.protocol)) {
throw new ProviderValidationError(`Invalid protocol "${url.protocol}"`, {
check: 'url',
details: {
protocol: url.protocol,
validProtocols
}
});
}
}
async initTransfer() {
const query = this.dispatcher?.dispatchCommand({
command: 'init'
});
const res = await query;
if (!res?.transferID) {
throw new ProviderTransferError('Init failed, invalid response from the server');
}
return res.transferID;
}
async bootstrap(diagnostics) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics] = diagnostics;
const { url, auth } = this.options;
let ws;
this.assertValidProtocol(url);
const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
const wsUrl = `${wsProtocol}//${url.host}${trimTrailingSlash(url.pathname)}${TRANSFER_PATH}/pull`;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('establishing websocket connection');
// No auth defined, trying public access for transfer
if (!auth) {
ws = await connectToWebsocket(wsUrl, undefined, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else if (auth.type === 'token') {
const headers = {
Authorization: `Bearer ${auth.token}`
};
ws = await connectToWebsocket(wsUrl, {
headers
}, _class_private_field_loose_base(this, _diagnostics)[_diagnostics]);
} else {
throw new ProviderValidationError('Auth method not available', {
check: 'auth.type',
details: {
auth: auth.type
}
});
}
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('established websocket connection');
this.ws = ws;
const { retryMessageOptions } = this.options;
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating dispatcher');
this.dispatcher = createDispatcher(this.ws, retryMessageOptions, (message)=>_class_private_field_loose_base(this, _reportInfo)[_reportInfo](message));
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('creating dispatcher');
_class_private_field_loose_base(this, _reportInfo)[_reportInfo]('initialize transfer');
const transferID = await this.initTransfer();
_class_private_field_loose_base(this, _reportInfo)[_reportInfo](`initialized transfer ${transferID}`);
this.dispatcher.setTransferProperties({
id: transferID,
kind: 'pull'
});
await this.dispatcher.dispatchTransferAction('bootstrap');
}
async close() {
await this.dispatcher?.dispatchTransferAction('close');
await new Promise((resolve)=>{
const { ws } = this;
if (!ws || ws.CLOSED) {
resolve();
return;
}
ws.on('close', ()=>resolve()).close();
});
}
async getSchemas() {
const schemas = await this.dispatcher?.dispatchTransferAction('getSchemas');
return schemas ?? null;
}
constructor(options){
Object.defineProperty(this, _createStageReadStream, {
value: createStageReadStream
});
Object.defineProperty(this, _reportInfo, {
value: reportInfo
});
Object.defineProperty(this, _startStep, {
value: startStep
});
Object.defineProperty(this, _respond, {
value: respond
});
Object.defineProperty(this, _endStep, {
value: endStep
});
Object.defineProperty(this, _diagnostics, {
writable: true,
value: void 0
});
this.name = 'source::remote-strapi';
this.type = 'source';
this.defaultOptions = {
streamTimeout: 15000
};
this.writeAsync = (stream, data)=>{
return new Promise((resolve, reject)=>{
stream.write(data, (error)=>{
if (error) {
reject(error);
}
resolve();
});
});
};
this.options = {
...this.defaultOptions,
...options
};
this.ws = null;
this.dispatcher = null;
}
}
async function createStageReadStream(stage) {
const startResult = await _class_private_field_loose_base(this, _startStep)[_startStep](stage);
if (startResult instanceof Error) {
throw startResult;
}
const { id: processID } = startResult;
const stream = new PassThrough({
objectMode: true
});
const listener = async (raw)=>{
const parsed = JSON.parse(raw.toString());
// If not a message related to our transfer process, ignore it
if (!parsed.uuid || parsed?.data?.type !== 'transfer' || parsed?.data?.id !== processID) {
this.ws?.once('message', listener);
return;
}
const { uuid, data: message } = parsed;
const { ended, error, data } = message;
if (error) {
await _class_private_field_loose_base(this, _respond)[_respond](uuid);
stream.destroy(error);
return;
}
if (ended) {
await _class_private_field_loose_base(this, _respond)[_respond](uuid);
await _class_private_field_loose_base(this, _endStep)[_endStep](stage);
stream.end();
return;
}
// if we get a single items instead of a batch
for (const item of castArray(data)){
stream.push(item);
}
this.ws?.once('message', listener);
await _class_private_field_loose_base(this, _respond)[_respond](uuid);
};
this.ws?.once('message', listener);
return stream;
}
function reportInfo(message) {
_class_private_field_loose_base(this, _diagnostics)[_diagnostics]?.report({
details: {
createdAt: new Date(),
message,
origin: 'remote-source-provider'
},
kind: 'info'
});
}
async function startStep(step) {
try {
return await this.dispatcher?.dispatchTransferStep({
action: 'start',
step
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new ProviderTransferError(e);
}
return new ProviderTransferError('Unexpected error');
}
}
async function respond(uuid) {
return new Promise((resolve, reject)=>{
this.ws?.send(JSON.stringify({
uuid
}), (e)=>{
if (e) {
reject(e);
} else {
resolve(e);
}
});
});
}
async function endStep(step) {
try {
await this.dispatcher?.dispatchTransferStep({
action: 'end',
step
});
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new ProviderTransferError(e);
}
return new ProviderTransferError('Unexpected error');
}
return null;
}
const createRemoteStrapiSourceProvider = (options)=>{
return new RemoteStrapiSourceProvider(options);
};
export { createRemoteStrapiSourceProvider };
//# sourceMappingURL=index.mjs.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,43 @@
import { WebSocket } from 'ws';
import type { Client } from '../../../types/remote/protocol';
import { IDiagnosticReporter } from '../../utils/diagnostic';
interface IDispatcherState {
transfer?: {
kind: Client.TransferKind;
id: string;
};
}
interface IDispatchOptions {
attachTransfer?: boolean;
}
type Dispatch<T> = Omit<T, 'transferID' | 'uuid'>;
export declare const createDispatcher: (ws: WebSocket, retryMessageOptions?: {
retryMessageMaxRetries: number;
retryMessageTimeout: number;
}, reportInfo?: (message: string) => void) => {
readonly transferID: string | undefined;
readonly transferKind: any;
setTransferProperties: (properties: Exclude<IDispatcherState['transfer'], undefined>) => void;
dispatch: <U = null>(message: Dispatch<Client.Message>, options?: IDispatchOptions) => Promise<U | null>;
dispatchCommand: <U_1 extends "end" | "init" | "status">(payload: {
command: U_1;
} & ([Client.GetCommandParams<U_1>] extends [never] ? unknown : {
params?: Client.GetCommandParams<U_1> | undefined;
})) => Promise<null>;
dispatchTransferAction: <T>(action: Client.Action['action']) => Promise<T | null>;
dispatchTransferStep: <T_1, A extends "end" | "start" | "stream" = "end" | "start" | "stream", S extends "entities" | "links" | "assets" | "configuration" = "entities" | "links" | "assets" | "configuration">(payload: {
step: S;
action: A;
} & (A extends 'stream' ? {
data: Client.GetTransferPushStreamData<S>;
} : unknown)) => Promise<T_1 | null>;
};
type WebsocketParams = ConstructorParameters<typeof WebSocket>;
type Address = WebsocketParams[0];
type Options = WebsocketParams[2];
export declare const connectToWebsocket: (address: Address, options?: Options, diagnostics?: IDiagnosticReporter) => Promise<WebSocket>;
export declare const trimTrailingSlash: (input: string) => string;
export declare const wait: (ms: number) => Promise<void>;
export declare const waitUntil: (test: () => boolean, interval: number) => Promise<void>;
export {};
//# sourceMappingURL=utils.d.ts.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/strapi/providers/utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAW,SAAS,EAAE,MAAM,IAAI,CAAC;AAExC,OAAO,KAAK,EAAE,MAAM,EAAU,MAAM,gCAAgC,CAAC;AASrE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,UAAU,gBAAgB;IACxB,QAAQ,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;CACtD;AAED,UAAU,gBAAgB;IACxB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,KAAK,QAAQ,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,CAAC,CAAC;AAElD,eAAO,MAAM,gBAAgB,OACvB,SAAS;;;gBAKA,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI;;;wCAgIxB,QAAQ,gBAAgB,CAAC,UAAU,CAAC,EAAE,SAAS,CAAC,KAC3D,IAAI;sEAzHI,gBAAgB,KACxB,QAAQ,CAAC,GAAG,IAAI,CAAC;;;;;;wCA6F6B,OAAO,MAAM,CAAC,QAAQ,CAAC;6NAW7D;QACP,IAAI,EAAE,CAAC,CAAC;QACR,MAAM,EAAE,CAAC,CAAC;KACX,GAAG,CAAC,CAAC,SAAS,QAAQ,GAAG;QAAE,IAAI,EAAE,OAAO,yBAAyB,CAAC,CAAC,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC;CAiCrF,CAAC;AAEF,KAAK,eAAe,GAAG,qBAAqB,CAAC,OAAO,SAAS,CAAC,CAAC;AAC/D,KAAK,OAAO,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AAClC,KAAK,OAAO,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;AAElC,eAAO,MAAM,kBAAkB,YACpB,OAAO,YACN,OAAO,gBACH,mBAAmB,KAChC,QAAQ,SAAS,CA0DnB,CAAC;AAEF,eAAO,MAAM,iBAAiB,UAAW,MAAM,KAAG,MAEjD,CAAC;AAEF,eAAO,MAAM,IAAI,OAAQ,MAAM,kBAI9B,CAAC;AAEF,eAAO,MAAM,SAAS,SAAgB,MAAM,OAAO,YAAY,MAAM,KAAG,QAAQ,IAAI,CAMnF,CAAC"}

View File

@@ -0,0 +1,173 @@
'use strict';
var crypto = require('crypto');
var ws = require('ws');
var providers = require('../../errors/providers.js');
const createDispatcher = (ws, retryMessageOptions = {
retryMessageMaxRetries: 5,
retryMessageTimeout: 30000
}, reportInfo)=>{
const state = {};
const dispatch = async (message, options = {})=>{
if (!ws) {
throw new Error('No websocket connection found');
}
return new Promise((resolve, reject)=>{
const uuid = crypto.randomUUID();
const payload = {
...message,
uuid
};
let numberOfTimesMessageWasSent = 0;
if (options.attachTransfer) {
Object.assign(payload, {
transferID: state.transfer?.id
});
}
if (message.type === 'command') {
reportInfo?.(`dispatching message command:${message.command} uuid:${uuid} sent:${numberOfTimesMessageWasSent}`);
} else if (message.type === 'transfer') {
const messageToSend = message;
reportInfo?.(`dispatching message action:${messageToSend.action} ${messageToSend.kind === 'step' ? `step:${messageToSend.step}` : ''} uuid:${uuid} sent:${numberOfTimesMessageWasSent}`);
}
const stringifiedPayload = JSON.stringify(payload);
ws.send(stringifiedPayload, (error)=>{
if (error) {
reject(error);
}
});
const { retryMessageMaxRetries, retryMessageTimeout } = retryMessageOptions;
const sendPeriodically = ()=>{
if (numberOfTimesMessageWasSent <= retryMessageMaxRetries) {
numberOfTimesMessageWasSent += 1;
ws.send(stringifiedPayload, (error)=>{
if (error) {
reject(error);
}
});
} else {
reject(new providers.ProviderError('error', 'Request timed out'));
}
};
const interval = setInterval(sendPeriodically, retryMessageTimeout);
const onResponse = (raw)=>{
const response = JSON.parse(raw.toString());
if (message.type === 'command') {
reportInfo?.(`received response to message command: ${message.command} uuid: ${uuid} sent: ${numberOfTimesMessageWasSent}`);
} else if (message.type === 'transfer') {
const messageToSend = message;
reportInfo?.(`received response to message action:${messageToSend.action} ${messageToSend.kind === 'step' ? `step:${messageToSend.step}` : ''} uuid:${uuid} sent:${numberOfTimesMessageWasSent}`);
}
if (response.uuid === uuid) {
clearInterval(interval);
if (response.error) {
const message = response.error.message;
const details = response.error.details?.details;
const step = response.error.details?.step;
let error = new providers.ProviderError('error', message, details);
if (step === 'transfer') {
error = new providers.ProviderTransferError(message, details);
} else if (step === 'validation') {
error = new providers.ProviderValidationError(message, details);
} else if (step === 'initialization') {
error = new providers.ProviderInitializationError(message);
}
return reject(error);
}
resolve(response.data ?? null);
} else {
ws.once('message', onResponse);
}
};
ws.once('message', onResponse);
});
};
const dispatchCommand = (payload)=>{
return dispatch({
type: 'command',
...payload
});
};
const dispatchTransferAction = async (action)=>{
const payload = {
type: 'transfer',
kind: 'action',
action
};
return dispatch(payload, {
attachTransfer: true
}) ?? Promise.resolve(null);
};
const dispatchTransferStep = async (payload)=>{
const message = {
type: 'transfer',
kind: 'step',
...payload
};
return dispatch(message, {
attachTransfer: true
}) ?? Promise.resolve(null);
};
const setTransferProperties = (properties)=>{
state.transfer = {
...properties
};
};
return {
get transferID () {
return state.transfer?.id;
},
get transferKind () {
return state.transfer?.kind;
},
setTransferProperties,
dispatch,
dispatchCommand,
dispatchTransferAction,
dispatchTransferStep
};
};
const connectToWebsocket = (address, options, diagnostics)=>{
return new Promise((resolve, reject)=>{
const server = new ws.WebSocket(address, options);
server.once('open', ()=>{
resolve(server);
});
server.on('unexpected-response', (_req, res)=>{
if (res.statusCode === 401) {
return reject(new providers.ProviderInitializationError('Failed to initialize the connection: Authentication Error'));
}
if (res.statusCode === 403) {
return reject(new providers.ProviderInitializationError('Failed to initialize the connection: Authorization Error'));
}
if (res.statusCode === 404) {
return reject(new providers.ProviderInitializationError('Failed to initialize the connection: Data transfer is not enabled on the remote host'));
}
return reject(new providers.ProviderInitializationError(`Failed to initialize the connection: Unexpected server response ${res.statusCode}`));
});
server.on('message', (raw)=>{
const response = JSON.parse(raw.toString());
if (response.diagnostic) {
diagnostics?.report({
...response.diagnostic
});
}
});
server.once('error', (err)=>{
reject(new providers.ProviderTransferError(err.message, {
details: {
error: err.message
}
}));
});
});
};
const trimTrailingSlash = (input)=>{
return input.replace(/\/$/, '');
};
exports.connectToWebsocket = connectToWebsocket;
exports.createDispatcher = createDispatcher;
exports.trimTrailingSlash = trimTrailingSlash;
//# sourceMappingURL=utils.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,169 @@
import { randomUUID } from 'crypto';
import { WebSocket } from 'ws';
import { ProviderInitializationError, ProviderTransferError, ProviderError, ProviderValidationError } from '../../errors/providers.mjs';
const createDispatcher = (ws, retryMessageOptions = {
retryMessageMaxRetries: 5,
retryMessageTimeout: 30000
}, reportInfo)=>{
const state = {};
const dispatch = async (message, options = {})=>{
if (!ws) {
throw new Error('No websocket connection found');
}
return new Promise((resolve, reject)=>{
const uuid = randomUUID();
const payload = {
...message,
uuid
};
let numberOfTimesMessageWasSent = 0;
if (options.attachTransfer) {
Object.assign(payload, {
transferID: state.transfer?.id
});
}
if (message.type === 'command') {
reportInfo?.(`dispatching message command:${message.command} uuid:${uuid} sent:${numberOfTimesMessageWasSent}`);
} else if (message.type === 'transfer') {
const messageToSend = message;
reportInfo?.(`dispatching message action:${messageToSend.action} ${messageToSend.kind === 'step' ? `step:${messageToSend.step}` : ''} uuid:${uuid} sent:${numberOfTimesMessageWasSent}`);
}
const stringifiedPayload = JSON.stringify(payload);
ws.send(stringifiedPayload, (error)=>{
if (error) {
reject(error);
}
});
const { retryMessageMaxRetries, retryMessageTimeout } = retryMessageOptions;
const sendPeriodically = ()=>{
if (numberOfTimesMessageWasSent <= retryMessageMaxRetries) {
numberOfTimesMessageWasSent += 1;
ws.send(stringifiedPayload, (error)=>{
if (error) {
reject(error);
}
});
} else {
reject(new ProviderError('error', 'Request timed out'));
}
};
const interval = setInterval(sendPeriodically, retryMessageTimeout);
const onResponse = (raw)=>{
const response = JSON.parse(raw.toString());
if (message.type === 'command') {
reportInfo?.(`received response to message command: ${message.command} uuid: ${uuid} sent: ${numberOfTimesMessageWasSent}`);
} else if (message.type === 'transfer') {
const messageToSend = message;
reportInfo?.(`received response to message action:${messageToSend.action} ${messageToSend.kind === 'step' ? `step:${messageToSend.step}` : ''} uuid:${uuid} sent:${numberOfTimesMessageWasSent}`);
}
if (response.uuid === uuid) {
clearInterval(interval);
if (response.error) {
const message = response.error.message;
const details = response.error.details?.details;
const step = response.error.details?.step;
let error = new ProviderError('error', message, details);
if (step === 'transfer') {
error = new ProviderTransferError(message, details);
} else if (step === 'validation') {
error = new ProviderValidationError(message, details);
} else if (step === 'initialization') {
error = new ProviderInitializationError(message);
}
return reject(error);
}
resolve(response.data ?? null);
} else {
ws.once('message', onResponse);
}
};
ws.once('message', onResponse);
});
};
const dispatchCommand = (payload)=>{
return dispatch({
type: 'command',
...payload
});
};
const dispatchTransferAction = async (action)=>{
const payload = {
type: 'transfer',
kind: 'action',
action
};
return dispatch(payload, {
attachTransfer: true
}) ?? Promise.resolve(null);
};
const dispatchTransferStep = async (payload)=>{
const message = {
type: 'transfer',
kind: 'step',
...payload
};
return dispatch(message, {
attachTransfer: true
}) ?? Promise.resolve(null);
};
const setTransferProperties = (properties)=>{
state.transfer = {
...properties
};
};
return {
get transferID () {
return state.transfer?.id;
},
get transferKind () {
return state.transfer?.kind;
},
setTransferProperties,
dispatch,
dispatchCommand,
dispatchTransferAction,
dispatchTransferStep
};
};
const connectToWebsocket = (address, options, diagnostics)=>{
return new Promise((resolve, reject)=>{
const server = new WebSocket(address, options);
server.once('open', ()=>{
resolve(server);
});
server.on('unexpected-response', (_req, res)=>{
if (res.statusCode === 401) {
return reject(new ProviderInitializationError('Failed to initialize the connection: Authentication Error'));
}
if (res.statusCode === 403) {
return reject(new ProviderInitializationError('Failed to initialize the connection: Authorization Error'));
}
if (res.statusCode === 404) {
return reject(new ProviderInitializationError('Failed to initialize the connection: Data transfer is not enabled on the remote host'));
}
return reject(new ProviderInitializationError(`Failed to initialize the connection: Unexpected server response ${res.statusCode}`));
});
server.on('message', (raw)=>{
const response = JSON.parse(raw.toString());
if (response.diagnostic) {
diagnostics?.report({
...response.diagnostic
});
}
});
server.once('error', (err)=>{
reject(new ProviderTransferError(err.message, {
details: {
error: err.message
}
}));
});
});
};
const trimTrailingSlash = (input)=>{
return input.replace(/\/$/, '');
};
export { connectToWebsocket, createDispatcher, trimTrailingSlash };
//# sourceMappingURL=utils.mjs.map

File diff suppressed because one or more lines are too long