From b0d4fecd8fc7bd04884d92cffc03f159008155cb Mon Sep 17 00:00:00 2001 From: David Ball Date: Wed, 21 Aug 2024 16:03:44 -0400 Subject: [PATCH] Added ads.txt and robots.txt back in, prevent crawling dev mode site, and update amazon-catalog app. --- package-lock.json | 8 +-- package.json | 2 +- src/apiclient/amazon.ts | 66 +++++++++++++++++++--- src/apps/amazon-catalog.ts | 5 +- src/apps/modules/amazon-catalog-helpers.ts | 8 ++- src/pages/ads.txt.ts | 16 ++++++ src/pages/robots.txt.ts | 23 ++++++++ 7 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 src/pages/ads.txt.ts create mode 100644 src/pages/robots.txt.ts diff --git a/package-lock.json b/package-lock.json index 2b97e93..2fecd27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "@strapi/blocks-react-renderer": "^1.0.1", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", - "amazon-pa-api5-node-ts": "^2.3.0", + "amazon-pa-api5-node-ts": "^2.3.2", "astro": "^4.14.2", "axios": "^1.7.4", "bootstrap": "^5.3.3", @@ -3237,9 +3237,9 @@ "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==" }, "node_modules/amazon-pa-api5-node-ts": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/amazon-pa-api5-node-ts/-/amazon-pa-api5-node-ts-2.3.0.tgz", - "integrity": "sha512-dnaBwdoNbSaNwmkaUEjU9qvjZ4l9dzDwu2BbALa8bdy5XCTXuvYLQrwga2M4xDvivK1ldFqR/llUmNrUeOKC0w==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/amazon-pa-api5-node-ts/-/amazon-pa-api5-node-ts-2.3.2.tgz", + "integrity": "sha512-1/l4C7prHCg/vCeYevoEyCkvVrdfGmWXmg9K6JK+hhskL4D2n3KYvnqkSdgVSvvqgqJjoWuQx7Ec53NuCcO7MA==", "dependencies": { "superagent": "^9.0.0" } diff --git a/package.json b/package.json index 0dca0d7..f981220 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "@strapi/blocks-react-renderer": "^1.0.1", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", - "amazon-pa-api5-node-ts": "^2.3.0", + "amazon-pa-api5-node-ts": "^2.3.2", "astro": "^4.14.2", "axios": "^1.7.4", "bootstrap": "^5.3.3", diff --git a/src/apiclient/amazon.ts b/src/apiclient/amazon.ts index 9d1da28..85d3c48 100644 --- a/src/apiclient/amazon.ts +++ b/src/apiclient/amazon.ts @@ -28,18 +28,70 @@ export const getItems = (itemIds: string[]): Promise((resolve, reject) => { - return api.getItems(getItemsRequest).then( + return api.getItems(ProductAdvertisingAPIv1.GetItemsRequest.constructFromObject(getItemsRequest)!).then( function(data) { console.log('API called successfully.'); const getItemsResponse = ProductAdvertisingAPIv1.GetItemsResponse.constructFromObject(data)!; diff --git a/src/apps/amazon-catalog.ts b/src/apps/amazon-catalog.ts index 97d259b..e2a4cd5 100644 --- a/src/apps/amazon-catalog.ts +++ b/src/apps/amazon-catalog.ts @@ -167,6 +167,7 @@ program.command('append-images') }) .configureHelp(); +// TODO: when I get back I need to build a command that lets me replay the procure-asins previous API request program.command('procure-asins') .alias('procure-asin') @@ -190,7 +191,7 @@ program.command('procure-asins') } const MAX_ITEMS_PER_API_REQUEST = 10; for (let a = 0; a < asins.length; a += MAX_ITEMS_PER_API_REQUEST) { - let asinsToRequest = asins.slice(a, a + MAX_ITEMS_PER_API_REQUEST).map(asin => asin = asin.toLowerCase()); + let asinsToRequest = asins.slice(a, a + MAX_ITEMS_PER_API_REQUEST).map(asin => asin = asin.toUpperCase()); console.log(`[procure-asin] Begin product enrollment(s) for ${asins.length} ASINs ${asinsToRequest.join(' ')}.`); //get Amazon data ; we will use previously obtained data for the testing phase @@ -269,7 +270,7 @@ program.command('procure-asins') console.log(`[procure-asin] Matching Brand ID: ${brandsDto.items[0].id}, Brand Name: ${brandsDto.items[0].data?.brandName['en-US']}.`); } let brandName = brandsDto.items[0].data?.brandName['en-US']!; - let features = amazonItem.ItemInfo?.Features?.DisplayValues!; + let features = amazonItem.ItemInfo?.Features?.DisplayValues!||[]; console.log(`[procure-asin] Amazon Product Name (en-US): ${amazonProductName_en_US}`); console.log(`[procure-asin] Requesting llama3.1 LLM to translate product name from en-US to es-US.`); diff --git a/src/apps/modules/amazon-catalog-helpers.ts b/src/apps/modules/amazon-catalog-helpers.ts index 916386d..c9c6606 100644 --- a/src/apps/modules/amazon-catalog-helpers.ts +++ b/src/apps/modules/amazon-catalog-helpers.ts @@ -39,9 +39,11 @@ export async function lookupAmazonASINs(asins: string[]) { // let apiResponse = response as GetItemsResponse; let amazonGetItem: AmazonGetItem = { apiResponse: { iv: apiResponse }, - getItemsRequest: { iv: { schemaId: amazonGetItemsRequestSchemaDto.id, ItemIds: [ { - ItemId: (apiResponse.ItemsResult && apiResponse.ItemsResult.Items!.length > 0) ? apiResponse.ItemsResult!.Items![0].ASIN! : asins[0] - } ] } }, + getItemsRequest: { iv: { schemaId: amazonGetItemsRequestSchemaDto.id, ItemIds: [ + ...((apiResponse.ItemsResult && apiResponse.ItemsResult.Items!.length > 0) ? + apiResponse.ItemsResult!.Items!.map((item) => { return { ItemId: item.ASIN! } }) : + asins.map((asin) => { return { ItemId: asin }})) + ] } }, requestDate: { iv: requestDate } } let amazonGetItemDto = await client.contents.postContent(SCHEMAS.AMAZON_GET_ITEMS, { diff --git a/src/pages/ads.txt.ts b/src/pages/ads.txt.ts new file mode 100644 index 0000000..f92cb4e --- /dev/null +++ b/src/pages/ads.txt.ts @@ -0,0 +1,16 @@ +import type { APIRoute } from 'astro'; +import { config } from "../config"; +import { getSiteConfig } from '../data/api-client'; + +const siteConfigDto = await getSiteConfig(); +const siteConfig = siteConfigDto.items[0].data!; + +const adsTxt = siteConfig.googleAdSense.iv.adsTxt; + +export const GET: APIRoute = () => { + return new Response(adsTxt, { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + }, + }); +}; \ No newline at end of file diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts new file mode 100644 index 0000000..f6ecf74 --- /dev/null +++ b/src/pages/robots.txt.ts @@ -0,0 +1,23 @@ +import type { APIRoute } from 'astro'; + +const shouldPermitRobots = import.meta.env.MODE !== 'development'; + +const permissiveRobotsTxt = ` +User-agent: * +Allow: / + +Sitemap: ${new URL('sitemap-index.xml', import.meta.env.SITE).href} +`.trimStart(); + +const restrictiveRobotsTxt = ` +User-agent: * +Disallow: / +`.trimStart(); + +export const GET: APIRoute = () => { + return new Response(shouldPermitRobots ? permissiveRobotsTxt : restrictiveRobotsTxt, { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + }, + }); +}; \ No newline at end of file