removed gcs credentials
This commit is contained in:
parent
bd868852df
commit
e095df4b54
28
bqexport.js
28
bqexport.js
|
|
@ -1,15 +1,15 @@
|
|||
import { BigQuery } from '@google-cloud/bigquery';
|
||||
import { Storage } from '@google-cloud/storage';
|
||||
import { BigQuery } from "@google-cloud/bigquery";
|
||||
import { Storage } from "@google-cloud/storage";
|
||||
|
||||
// Cross-project setup:
|
||||
// - BigQuery data source: fynd-jio-commerceml-prod
|
||||
// - GCS destination: fynd-boltic-prod
|
||||
const bigquery = new BigQuery({
|
||||
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod'
|
||||
projectId: process.env.PROJECT_ID || "fynd-jio-commerceml-prod",
|
||||
});
|
||||
const storage = new Storage({
|
||||
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod',
|
||||
credentials: process.env.GCS_CREDENTIALS || {}
|
||||
projectId: process.env.PROJECT_ID || "fynd-jio-commerceml-prod",
|
||||
credentials: {}
|
||||
});
|
||||
|
||||
export const exportMergedJson = async (req, res) => {
|
||||
|
|
@ -19,13 +19,12 @@ export const exportMergedJson = async (req, res) => {
|
|||
|
||||
const datasetId = process.env.DATASET || "temp_zenith_data";
|
||||
const sourceTable = process.env.SOURCE_TABLE || "pr_training_data";
|
||||
const bucketName = process.env.BUCKET_NAME || 'pr_dataset_storage';
|
||||
const bucketName = process.env.BUCKET_NAME || "pr_dataset_storage";
|
||||
const objectPath = `${companyIdToDelete}/${applicationIdToDelete}/GET_ALL_PRODUCTS/catalog-*.json`;
|
||||
const uri = `gs://${bucketName}/${objectPath}`;
|
||||
|
||||
// Get deletion criteria from request body or environment
|
||||
|
||||
|
||||
// 1️⃣ Direct export from source table to GCS (cross-project)
|
||||
const sourceTablePath = `${bigquery.projectId}.${datasetId}.${sourceTable}`;
|
||||
|
||||
|
|
@ -70,15 +69,18 @@ export const exportMergedJson = async (req, res) => {
|
|||
// }
|
||||
|
||||
const options = {
|
||||
version: 'v4',
|
||||
action: 'read',
|
||||
version: "v4",
|
||||
action: "read",
|
||||
expires: Date.now() + 15 * 60 * 1000,
|
||||
extensionHeaders: {},
|
||||
queryParams: {},
|
||||
method: 'POST'
|
||||
method: "POST",
|
||||
};
|
||||
|
||||
const signedUrl = await storage.bucket(bucketName).file(objectPath.replace('*', '000000000000')).getSignedUrl(options);
|
||||
const signedUrl = await storage
|
||||
.bucket(bucketName)
|
||||
.file(objectPath.replace("*", "000000000000"))
|
||||
.getSignedUrl(options);
|
||||
|
||||
return {
|
||||
signedUrl,
|
||||
|
|
@ -86,10 +88,10 @@ export const exportMergedJson = async (req, res) => {
|
|||
message: `Success: exported to ${uri}`,
|
||||
companyId: companyIdToDelete,
|
||||
applicationId: applicationIdToDelete,
|
||||
destination: uri
|
||||
destination: uri,
|
||||
};
|
||||
} catch (err) {
|
||||
console.error('Export or cleanup failed:', err);
|
||||
console.error("Export or cleanup failed:", err);
|
||||
throw new Error(`Export failed: ${err.message}`);
|
||||
}
|
||||
};
|
||||
3
index.js
3
index.js
|
|
@ -1,11 +1,14 @@
|
|||
import express from 'express';
|
||||
import { exportMergedJson } from './bqexport.js';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
const app = express();
|
||||
|
||||
// Middleware to parse JSON bodies
|
||||
app.use(express.json());
|
||||
|
||||
dotenv.config({ path: '.env' });
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/', (req, res) => {
|
||||
res.json({
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
"dependencies": {
|
||||
"@google-cloud/bigquery": "^6.0.0",
|
||||
"@google-cloud/storage": "^7.16.0",
|
||||
"dotenv": "^17.2.1",
|
||||
"express": "^4.21.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -639,6 +640,17 @@
|
|||
"npm": "1.2.8000 || >= 1.4.16"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "17.2.1",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.1.tgz",
|
||||
"integrity": "sha512-kQhDYKZecqnM0fCnzI5eIv5L4cAe/iRI+HqMbO/hbRdTAeXDG+M9FjipUxNfbARuEg4iHIbhnhs78BCHNbSxEQ==",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://dotenvx.com"
|
||||
}
|
||||
},
|
||||
"node_modules/dunder-proto": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -12,9 +12,10 @@
|
|||
"node": ">=18.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"express": "^4.21.2",
|
||||
"@google-cloud/bigquery": "^6.0.0",
|
||||
"@google-cloud/storage": "^7.16.0"
|
||||
"@google-cloud/storage": "^7.16.0",
|
||||
"dotenv": "^17.2.1",
|
||||
"express": "^4.21.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.1.10"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user