removed gcs credentials

This commit is contained in:
Raunak Singh 2025-07-31 12:20:30 +05:30
parent bd868852df
commit e095df4b54
4 changed files with 34 additions and 16 deletions

View File

@ -1,15 +1,15 @@
import { BigQuery } from '@google-cloud/bigquery';
import { Storage } from '@google-cloud/storage';
import { BigQuery } from "@google-cloud/bigquery";
import { Storage } from "@google-cloud/storage";
// Cross-project setup:
// - BigQuery data source: fynd-jio-commerceml-prod
// - GCS destination: fynd-boltic-prod
const bigquery = new BigQuery({
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod'
projectId: process.env.PROJECT_ID || "fynd-jio-commerceml-prod",
});
const storage = new Storage({
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod',
credentials: process.env.GCS_CREDENTIALS || {}
projectId: process.env.PROJECT_ID || "fynd-jio-commerceml-prod",
credentials: {}
});
export const exportMergedJson = async (req, res) => {
@ -19,13 +19,12 @@ export const exportMergedJson = async (req, res) => {
const datasetId = process.env.DATASET || "temp_zenith_data";
const sourceTable = process.env.SOURCE_TABLE || "pr_training_data";
const bucketName = process.env.BUCKET_NAME || 'pr_dataset_storage';
const bucketName = process.env.BUCKET_NAME || "pr_dataset_storage";
const objectPath = `${companyIdToDelete}/${applicationIdToDelete}/GET_ALL_PRODUCTS/catalog-*.json`;
const uri = `gs://${bucketName}/${objectPath}`;
// Get deletion criteria from request body or environment
// 1⃣ Direct export from source table to GCS (cross-project)
const sourceTablePath = `${bigquery.projectId}.${datasetId}.${sourceTable}`;
@ -70,15 +69,18 @@ export const exportMergedJson = async (req, res) => {
// }
const options = {
version: 'v4',
action: 'read',
version: "v4",
action: "read",
expires: Date.now() + 15 * 60 * 1000,
extensionHeaders: {},
queryParams: {},
method: 'POST'
method: "POST",
};
const signedUrl = await storage.bucket(bucketName).file(objectPath.replace('*', '000000000000')).getSignedUrl(options);
const signedUrl = await storage
.bucket(bucketName)
.file(objectPath.replace("*", "000000000000"))
.getSignedUrl(options);
return {
signedUrl,
@ -86,10 +88,10 @@ export const exportMergedJson = async (req, res) => {
message: `Success: exported to ${uri}`,
companyId: companyIdToDelete,
applicationId: applicationIdToDelete,
destination: uri
destination: uri,
};
} catch (err) {
console.error('Export or cleanup failed:', err);
console.error("Export or cleanup failed:", err);
throw new Error(`Export failed: ${err.message}`);
}
};
};

View File

@ -1,11 +1,14 @@
import express from 'express';
import { exportMergedJson } from './bqexport.js';
import dotenv from 'dotenv';
const app = express();
// Middleware to parse JSON bodies
app.use(express.json());
dotenv.config({ path: '.env' });
// Health check endpoint
app.get('/', (req, res) => {
res.json({

12
package-lock.json generated
View File

@ -10,6 +10,7 @@
"dependencies": {
"@google-cloud/bigquery": "^6.0.0",
"@google-cloud/storage": "^7.16.0",
"dotenv": "^17.2.1",
"express": "^4.21.2"
},
"devDependencies": {
@ -639,6 +640,17 @@
"npm": "1.2.8000 || >= 1.4.16"
}
},
"node_modules/dotenv": {
"version": "17.2.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.1.tgz",
"integrity": "sha512-kQhDYKZecqnM0fCnzI5eIv5L4cAe/iRI+HqMbO/hbRdTAeXDG+M9FjipUxNfbARuEg4iHIbhnhs78BCHNbSxEQ==",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",

View File

@ -12,9 +12,10 @@
"node": ">=18.0.0"
},
"dependencies": {
"express": "^4.21.2",
"@google-cloud/bigquery": "^6.0.0",
"@google-cloud/storage": "^7.16.0"
"@google-cloud/storage": "^7.16.0",
"dotenv": "^17.2.1",
"express": "^4.21.2"
},
"devDependencies": {
"nodemon": "^3.1.10"