Refactor bqexport.js to use environment variables for project IDs and bucket name; update object path to include dynamic company and application IDs. Modify index.js to export app for serverless functions and update package.json description and scripts for better clarity.

This commit is contained in:
Raunak Singh 2025-07-31 11:36:25 +05:30
parent 6764412587
commit 4f12397489
4 changed files with 58 additions and 10 deletions

42
Dockerfile Normal file
View File

@ -0,0 +1,42 @@
FROM node:18-alpine AS builder
# Create app directory
RUN mkdir -p /app
WORKDIR /app
# Copy package files first for better caching
COPY package*.json ./
# Install dependencies
RUN npm ci --only=production
# Copy source code
COPY . .
# Production stage
FROM node:18-alpine
# Create app directory
RUN mkdir -p /app
WORKDIR /app
# Copy built application from builder stage
COPY --from=builder /app .
# Create non-root user for security
RUN addgroup -g 1001 -S nodejs && \
adduser -S nodejs -u 1001
# Change ownership of the app directory to nodejs user
RUN chown -R nodejs:nodejs /app
USER nodejs
# Expose the port the app runs on
EXPOSE 8080
# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
CMD node -e "require('http').get('http://localhost:8080/', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) })"
# Start the application
CMD ["npm", "start"]

View File

@ -5,24 +5,26 @@ import { Storage } from '@google-cloud/storage';
// - BigQuery data source: fynd-jio-commerceml-prod
// - GCS destination: fynd-boltic-prod
const bigquery = new BigQuery({
projectId: process.env.BIGQUERY_PROJECT || 'fynd-jio-commerceml-prod'
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod'
});
const storage = new Storage({
projectId: process.env.STORAGE_PROJECT || 'fynd-jio-commerceml-prod',
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod',
credentials: {}
});
export const exportMergedJson = async (req, res) => {
try {
const companyIdToDelete = req.body?.companyId;
const applicationIdToDelete = req.body?.applicationId;
const datasetId = process.env.DATASET || "temp_zenith_data";
const sourceTable = process.env.SOURCE_TABLE || "pr_training_data";
const bucketName = 'pr_dataset_storage';
const objectPath = '261/643e93f5ffc8101656a5629b/GET_ALL_PRODUCTS/catalog-*.json';
const bucketName = process.env.BUCKET_NAME || 'pr_dataset_storage';
const objectPath = `${companyIdToDelete}/${applicationIdToDelete}/GET_ALL_PRODUCTS/catalog-*.json`;
const uri = `gs://${bucketName}/${objectPath}`;
// Get deletion criteria from request body or environment
const companyIdToDelete = req.body?.companyId;
const applicationIdToDelete = req.body?.applicationId;
// 1⃣ Direct export from source table to GCS (cross-project)
const sourceTablePath = `${bigquery.projectId}.${datasetId}.${sourceTable}`;

View File

@ -49,4 +49,8 @@ app.post('/export', async (req, res) => {
const PORT = process.env.PORT || 8080;
app.listen(PORT, () => {
console.log(`Server is running on port ${PORT}`);
});
});
// For serverless functions (like Google Cloud Functions)
export default app;

View File

@ -2,11 +2,11 @@
"type": "module",
"name": "bq-exporter",
"version": "1.0.0",
"description": "Express.js web server with dynamic greetings",
"description": "BigQuery export serverless function",
"main": "index.js",
"scripts": {
"start": "node index.js",
"dev": "nodemon index.js"
"start": "nodemon index.js",
"dev": "LOCAL_DEV=falsetrue nodemon index.js"
},
"engines": {
"node": ">=18.0.0"