removed gcs credentials
This commit is contained in:
parent
bd868852df
commit
e095df4b54
30
bqexport.js
30
bqexport.js
|
|
@ -1,15 +1,15 @@
|
||||||
import { BigQuery } from '@google-cloud/bigquery';
|
import { BigQuery } from "@google-cloud/bigquery";
|
||||||
import { Storage } from '@google-cloud/storage';
|
import { Storage } from "@google-cloud/storage";
|
||||||
|
|
||||||
// Cross-project setup:
|
// Cross-project setup:
|
||||||
// - BigQuery data source: fynd-jio-commerceml-prod
|
// - BigQuery data source: fynd-jio-commerceml-prod
|
||||||
// - GCS destination: fynd-boltic-prod
|
// - GCS destination: fynd-boltic-prod
|
||||||
const bigquery = new BigQuery({
|
const bigquery = new BigQuery({
|
||||||
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod'
|
projectId: process.env.PROJECT_ID || "fynd-jio-commerceml-prod",
|
||||||
});
|
});
|
||||||
const storage = new Storage({
|
const storage = new Storage({
|
||||||
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod',
|
projectId: process.env.PROJECT_ID || "fynd-jio-commerceml-prod",
|
||||||
credentials: process.env.GCS_CREDENTIALS || {}
|
credentials: {}
|
||||||
});
|
});
|
||||||
|
|
||||||
export const exportMergedJson = async (req, res) => {
|
export const exportMergedJson = async (req, res) => {
|
||||||
|
|
@ -19,13 +19,12 @@ export const exportMergedJson = async (req, res) => {
|
||||||
|
|
||||||
const datasetId = process.env.DATASET || "temp_zenith_data";
|
const datasetId = process.env.DATASET || "temp_zenith_data";
|
||||||
const sourceTable = process.env.SOURCE_TABLE || "pr_training_data";
|
const sourceTable = process.env.SOURCE_TABLE || "pr_training_data";
|
||||||
const bucketName = process.env.BUCKET_NAME || 'pr_dataset_storage';
|
const bucketName = process.env.BUCKET_NAME || "pr_dataset_storage";
|
||||||
const objectPath = `${companyIdToDelete}/${applicationIdToDelete}/GET_ALL_PRODUCTS/catalog-*.json`;
|
const objectPath = `${companyIdToDelete}/${applicationIdToDelete}/GET_ALL_PRODUCTS/catalog-*.json`;
|
||||||
const uri = `gs://${bucketName}/${objectPath}`;
|
const uri = `gs://${bucketName}/${objectPath}`;
|
||||||
|
|
||||||
// Get deletion criteria from request body or environment
|
// Get deletion criteria from request body or environment
|
||||||
|
|
||||||
|
|
||||||
// 1️⃣ Direct export from source table to GCS (cross-project)
|
// 1️⃣ Direct export from source table to GCS (cross-project)
|
||||||
const sourceTablePath = `${bigquery.projectId}.${datasetId}.${sourceTable}`;
|
const sourceTablePath = `${bigquery.projectId}.${datasetId}.${sourceTable}`;
|
||||||
|
|
||||||
|
|
@ -70,15 +69,18 @@ export const exportMergedJson = async (req, res) => {
|
||||||
// }
|
// }
|
||||||
|
|
||||||
const options = {
|
const options = {
|
||||||
version: 'v4',
|
version: "v4",
|
||||||
action: 'read',
|
action: "read",
|
||||||
expires: Date.now() + 15 * 60 * 1000,
|
expires: Date.now() + 15 * 60 * 1000,
|
||||||
extensionHeaders: {},
|
extensionHeaders: {},
|
||||||
queryParams: {},
|
queryParams: {},
|
||||||
method: 'POST'
|
method: "POST",
|
||||||
};
|
};
|
||||||
|
|
||||||
const signedUrl = await storage.bucket(bucketName).file(objectPath.replace('*', '000000000000')).getSignedUrl(options);
|
const signedUrl = await storage
|
||||||
|
.bucket(bucketName)
|
||||||
|
.file(objectPath.replace("*", "000000000000"))
|
||||||
|
.getSignedUrl(options);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
signedUrl,
|
signedUrl,
|
||||||
|
|
@ -86,10 +88,10 @@ export const exportMergedJson = async (req, res) => {
|
||||||
message: `Success: exported to ${uri}`,
|
message: `Success: exported to ${uri}`,
|
||||||
companyId: companyIdToDelete,
|
companyId: companyIdToDelete,
|
||||||
applicationId: applicationIdToDelete,
|
applicationId: applicationIdToDelete,
|
||||||
destination: uri
|
destination: uri,
|
||||||
};
|
};
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('Export or cleanup failed:', err);
|
console.error("Export or cleanup failed:", err);
|
||||||
throw new Error(`Export failed: ${err.message}`);
|
throw new Error(`Export failed: ${err.message}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
||||||
3
index.js
3
index.js
|
|
@ -1,11 +1,14 @@
|
||||||
import express from 'express';
|
import express from 'express';
|
||||||
import { exportMergedJson } from './bqexport.js';
|
import { exportMergedJson } from './bqexport.js';
|
||||||
|
import dotenv from 'dotenv';
|
||||||
|
|
||||||
const app = express();
|
const app = express();
|
||||||
|
|
||||||
// Middleware to parse JSON bodies
|
// Middleware to parse JSON bodies
|
||||||
app.use(express.json());
|
app.use(express.json());
|
||||||
|
|
||||||
|
dotenv.config({ path: '.env' });
|
||||||
|
|
||||||
// Health check endpoint
|
// Health check endpoint
|
||||||
app.get('/', (req, res) => {
|
app.get('/', (req, res) => {
|
||||||
res.json({
|
res.json({
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@google-cloud/bigquery": "^6.0.0",
|
"@google-cloud/bigquery": "^6.0.0",
|
||||||
"@google-cloud/storage": "^7.16.0",
|
"@google-cloud/storage": "^7.16.0",
|
||||||
|
"dotenv": "^17.2.1",
|
||||||
"express": "^4.21.2"
|
"express": "^4.21.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
@ -639,6 +640,17 @@
|
||||||
"npm": "1.2.8000 || >= 1.4.16"
|
"npm": "1.2.8000 || >= 1.4.16"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/dotenv": {
|
||||||
|
"version": "17.2.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.1.tgz",
|
||||||
|
"integrity": "sha512-kQhDYKZecqnM0fCnzI5eIv5L4cAe/iRI+HqMbO/hbRdTAeXDG+M9FjipUxNfbARuEg4iHIbhnhs78BCHNbSxEQ==",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=12"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"url": "https://dotenvx.com"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/dunder-proto": {
|
"node_modules/dunder-proto": {
|
||||||
"version": "1.0.1",
|
"version": "1.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||||
|
|
|
||||||
|
|
@ -12,9 +12,10 @@
|
||||||
"node": ">=18.0.0"
|
"node": ">=18.0.0"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"express": "^4.21.2",
|
|
||||||
"@google-cloud/bigquery": "^6.0.0",
|
"@google-cloud/bigquery": "^6.0.0",
|
||||||
"@google-cloud/storage": "^7.16.0"
|
"@google-cloud/storage": "^7.16.0",
|
||||||
|
"dotenv": "^17.2.1",
|
||||||
|
"express": "^4.21.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"nodemon": "^3.1.10"
|
"nodemon": "^3.1.10"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user