Refactor bqexport.js to use environment variables for project IDs and bucket name; update object path to include dynamic company and application IDs. Modify index.js to export app for serverless functions and update package.json description and scripts for better clarity.
This commit is contained in:
parent
6764412587
commit
4f12397489
|
|
@ -0,0 +1,42 @@
|
||||||
|
FROM node:18-alpine AS builder
|
||||||
|
|
||||||
|
# Create app directory
|
||||||
|
RUN mkdir -p /app
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy package files first for better caching
|
||||||
|
COPY package*.json ./
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
RUN npm ci --only=production
|
||||||
|
|
||||||
|
# Copy source code
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Production stage
|
||||||
|
FROM node:18-alpine
|
||||||
|
|
||||||
|
# Create app directory
|
||||||
|
RUN mkdir -p /app
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy built application from builder stage
|
||||||
|
COPY --from=builder /app .
|
||||||
|
|
||||||
|
# Create non-root user for security
|
||||||
|
RUN addgroup -g 1001 -S nodejs && \
|
||||||
|
adduser -S nodejs -u 1001
|
||||||
|
|
||||||
|
# Change ownership of the app directory to nodejs user
|
||||||
|
RUN chown -R nodejs:nodejs /app
|
||||||
|
USER nodejs
|
||||||
|
|
||||||
|
# Expose the port the app runs on
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
# Health check
|
||||||
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
|
CMD node -e "require('http').get('http://localhost:8080/', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) })"
|
||||||
|
|
||||||
|
# Start the application
|
||||||
|
CMD ["npm", "start"]
|
||||||
14
bqexport.js
14
bqexport.js
|
|
@ -5,24 +5,26 @@ import { Storage } from '@google-cloud/storage';
|
||||||
// - BigQuery data source: fynd-jio-commerceml-prod
|
// - BigQuery data source: fynd-jio-commerceml-prod
|
||||||
// - GCS destination: fynd-boltic-prod
|
// - GCS destination: fynd-boltic-prod
|
||||||
const bigquery = new BigQuery({
|
const bigquery = new BigQuery({
|
||||||
projectId: process.env.BIGQUERY_PROJECT || 'fynd-jio-commerceml-prod'
|
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod'
|
||||||
});
|
});
|
||||||
const storage = new Storage({
|
const storage = new Storage({
|
||||||
projectId: process.env.STORAGE_PROJECT || 'fynd-jio-commerceml-prod',
|
projectId: process.env.PROJECT_ID || 'fynd-jio-commerceml-prod',
|
||||||
credentials: {}
|
credentials: {}
|
||||||
});
|
});
|
||||||
|
|
||||||
export const exportMergedJson = async (req, res) => {
|
export const exportMergedJson = async (req, res) => {
|
||||||
try {
|
try {
|
||||||
|
const companyIdToDelete = req.body?.companyId;
|
||||||
|
const applicationIdToDelete = req.body?.applicationId;
|
||||||
|
|
||||||
const datasetId = process.env.DATASET || "temp_zenith_data";
|
const datasetId = process.env.DATASET || "temp_zenith_data";
|
||||||
const sourceTable = process.env.SOURCE_TABLE || "pr_training_data";
|
const sourceTable = process.env.SOURCE_TABLE || "pr_training_data";
|
||||||
const bucketName = 'pr_dataset_storage';
|
const bucketName = process.env.BUCKET_NAME || 'pr_dataset_storage';
|
||||||
const objectPath = '261/643e93f5ffc8101656a5629b/GET_ALL_PRODUCTS/catalog-*.json';
|
const objectPath = `${companyIdToDelete}/${applicationIdToDelete}/GET_ALL_PRODUCTS/catalog-*.json`;
|
||||||
const uri = `gs://${bucketName}/${objectPath}`;
|
const uri = `gs://${bucketName}/${objectPath}`;
|
||||||
|
|
||||||
// Get deletion criteria from request body or environment
|
// Get deletion criteria from request body or environment
|
||||||
const companyIdToDelete = req.body?.companyId;
|
|
||||||
const applicationIdToDelete = req.body?.applicationId;
|
|
||||||
|
|
||||||
// 1️⃣ Direct export from source table to GCS (cross-project)
|
// 1️⃣ Direct export from source table to GCS (cross-project)
|
||||||
const sourceTablePath = `${bigquery.projectId}.${datasetId}.${sourceTable}`;
|
const sourceTablePath = `${bigquery.projectId}.${datasetId}.${sourceTable}`;
|
||||||
|
|
|
||||||
6
index.js
6
index.js
|
|
@ -49,4 +49,8 @@ app.post('/export', async (req, res) => {
|
||||||
const PORT = process.env.PORT || 8080;
|
const PORT = process.env.PORT || 8080;
|
||||||
app.listen(PORT, () => {
|
app.listen(PORT, () => {
|
||||||
console.log(`Server is running on port ${PORT}`);
|
console.log(`Server is running on port ${PORT}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// For serverless functions (like Google Cloud Functions)
|
||||||
|
export default app;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,11 @@
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"name": "bq-exporter",
|
"name": "bq-exporter",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"description": "Express.js web server with dynamic greetings",
|
"description": "BigQuery export serverless function",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node index.js",
|
"start": "nodemon index.js",
|
||||||
"dev": "nodemon index.js"
|
"dev": "LOCAL_DEV=falsetrue nodemon index.js"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=18.0.0"
|
"node": ">=18.0.0"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user