diff --git a/bqexport.js b/bqexport.js index 95ac8c9..6b0d681 100644 --- a/bqexport.js +++ b/bqexport.js @@ -1,15 +1,15 @@ -import { BigQuery } from '@google-cloud/bigquery'; -import { Storage } from '@google-cloud/storage'; +import { BigQuery } from "@google-cloud/bigquery"; +import { Storage } from "@google-cloud/storage"; // Cross-project setup: // - BigQuery data source: fynd-jio-commerceml-prod // - GCS destination: fynd-boltic-prod const bigquery = new BigQuery({ - projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod' + projectId: process.env.PROJECT_ID || "fynd-jio-commerceml-prod", }); const storage = new Storage({ - projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod', - credentials: process.env.GCS_CREDENTIALS || {} + projectId: process.env.PROJECT_ID || "fynd-jio-commerceml-prod", + credentials: {} }); export const exportMergedJson = async (req, res) => { @@ -19,13 +19,12 @@ export const exportMergedJson = async (req, res) => { const datasetId = process.env.DATASET || "temp_zenith_data"; const sourceTable = process.env.SOURCE_TABLE || "pr_training_data"; - const bucketName = process.env.BUCKET_NAME || 'pr_dataset_storage'; + const bucketName = process.env.BUCKET_NAME || "pr_dataset_storage"; const objectPath = `${companyIdToDelete}/${applicationIdToDelete}/GET_ALL_PRODUCTS/catalog-*.json`; const uri = `gs://${bucketName}/${objectPath}`; // Get deletion criteria from request body or environment - // 1️⃣ Direct export from source table to GCS (cross-project) const sourceTablePath = `${bigquery.projectId}.${datasetId}.${sourceTable}`; @@ -70,15 +69,18 @@ export const exportMergedJson = async (req, res) => { // } const options = { - version: 'v4', - action: 'read', + version: "v4", + action: "read", expires: Date.now() + 15 * 60 * 1000, extensionHeaders: {}, queryParams: {}, - method: 'POST' + method: "POST", }; - const signedUrl = await storage.bucket(bucketName).file(objectPath.replace('*', '000000000000')).getSignedUrl(options); + const signedUrl = await storage + .bucket(bucketName) + .file(objectPath.replace("*", "000000000000")) + .getSignedUrl(options); return { signedUrl, @@ -86,10 +88,10 @@ export const exportMergedJson = async (req, res) => { message: `Success: exported to ${uri}`, companyId: companyIdToDelete, applicationId: applicationIdToDelete, - destination: uri + destination: uri, }; } catch (err) { - console.error('Export or cleanup failed:', err); + console.error("Export or cleanup failed:", err); throw new Error(`Export failed: ${err.message}`); } -}; \ No newline at end of file +}; diff --git a/index.js b/index.js index 1516654..4aca90f 100644 --- a/index.js +++ b/index.js @@ -1,11 +1,14 @@ import express from 'express'; import { exportMergedJson } from './bqexport.js'; +import dotenv from 'dotenv'; const app = express(); // Middleware to parse JSON bodies app.use(express.json()); +dotenv.config({ path: '.env' }); + // Health check endpoint app.get('/', (req, res) => { res.json({ diff --git a/package-lock.json b/package-lock.json index 5582000..015b963 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@google-cloud/bigquery": "^6.0.0", "@google-cloud/storage": "^7.16.0", + "dotenv": "^17.2.1", "express": "^4.21.2" }, "devDependencies": { @@ -639,6 +640,17 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/dotenv": { + "version": "17.2.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.1.tgz", + "integrity": "sha512-kQhDYKZecqnM0fCnzI5eIv5L4cAe/iRI+HqMbO/hbRdTAeXDG+M9FjipUxNfbARuEg4iHIbhnhs78BCHNbSxEQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", diff --git a/package.json b/package.json index fc4d54c..5915c5a 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,10 @@ "node": ">=18.0.0" }, "dependencies": { - "express": "^4.21.2", "@google-cloud/bigquery": "^6.0.0", - "@google-cloud/storage": "^7.16.0" + "@google-cloud/storage": "^7.16.0", + "dotenv": "^17.2.1", + "express": "^4.21.2" }, "devDependencies": { "nodemon": "^3.1.10"