daball/dashersupply
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added ads.txt and robots.txt back in, prevent crawling dev mode site, and update amazon-catalog app.

Browse Source
This commit is contained in:
David Ball 2024-08-21 16:03:44 -04:00
parent 7354d923d0
commit b0d4fecd8f
7 changed files with 111 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
package-lock.json generated
View File

@ -18,7 +18,7 @@
"@strapi/blocks-react-renderer": "^1.0.1",
"@types/react": "^18.3.3",
"@types/react-dom": "^18.3.0",
"amazon-pa-api5-node-ts": "^2.3.0",
"amazon-pa-api5-node-ts": "^2.3.2",
"astro": "^4.14.2",
"axios": "^1.7.4",
"bootstrap": "^5.3.3",
@ -3237,9 +3237,9 @@
"integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw=="
},
"node_modules/amazon-pa-api5-node-ts": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/amazon-pa-api5-node-ts/-/amazon-pa-api5-node-ts-2.3.0.tgz",
"integrity": "sha512-dnaBwdoNbSaNwmkaUEjU9qvjZ4l9dzDwu2BbALa8bdy5XCTXuvYLQrwga2M4xDvivK1ldFqR/llUmNrUeOKC0w==",
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/amazon-pa-api5-node-ts/-/amazon-pa-api5-node-ts-2.3.2.tgz",
"integrity": "sha512-1/l4C7prHCg/vCeYevoEyCkvVrdfGmWXmg9K6JK+hhskL4D2n3KYvnqkSdgVSvvqgqJjoWuQx7Ec53NuCcO7MA==",
"dependencies": {
"superagent": "^9.0.0"
}

2
package.json
View File

@ -24,7 +24,7 @@
"@strapi/blocks-react-renderer": "^1.0.1",
"@types/react": "^18.3.3",
"@types/react-dom": "^18.3.0",
"amazon-pa-api5-node-ts": "^2.3.0",
"amazon-pa-api5-node-ts": "^2.3.2",
"astro": "^4.14.2",
"axios": "^1.7.4",
"bootstrap": "^5.3.3",

66
src/apiclient/amazon.ts
View File

@ -28,18 +28,70 @@ export const getItems = (itemIds: string[]): Promise<ProductAdvertisingAPIv1.Get
getItemsRequest['ItemIds'] = itemIds;
getItemsRequest['Condition'] = 'New';
getItemsRequest['Resources'] = [
GetItemsResourceValues['Images.Primary.Large'],
GetItemsResourceValues['Images.Variants.Large'],
GetItemsResourceValues['Offers.Listings.Price'],
GetItemsResourceValues['ItemInfo.Title'],
GetItemsResourceValues['ItemInfo.Features'],
GetItemsResourceValues['BrowseNodeInfo.BrowseNodes'],
GetItemsResourceValues['BrowseNodeInfo.BrowseNodes.Ancestor'],
GetItemsResourceValues['BrowseNodeInfo.BrowseNodes.SalesRank'],
GetItemsResourceValues['BrowseNodeInfo.WebsiteSalesRank'],
GetItemsResourceValues['CustomerReviews.Count'],
GetItemsResourceValues['CustomerReviews.StarRating'],
GetItemsResourceValues['ItemInfo.ProductInfo']
GetItemsResourceValues['Images.Primary.Small'],
GetItemsResourceValues['Images.Primary.Medium'],
GetItemsResourceValues['Images.Primary.Large'],
'Images.Primary.HighRes',
GetItemsResourceValues['Images.Variants.Small'],
GetItemsResourceValues['Images.Variants.Medium'],
GetItemsResourceValues['Images.Variants.Large'],
'Images.Variants.HighRes',
GetItemsResourceValues['ItemInfo.ByLineInfo'],
GetItemsResourceValues['ItemInfo.Classifications'],
GetItemsResourceValues['ItemInfo.ContentInfo'],
GetItemsResourceValues['ItemInfo.ContentRating'],
GetItemsResourceValues['ItemInfo.ExternalIds'],
GetItemsResourceValues['ItemInfo.Features'],
GetItemsResourceValues['ItemInfo.ManufactureInfo'],
GetItemsResourceValues['ItemInfo.ProductInfo'],
GetItemsResourceValues['ItemInfo.TechnicalInfo'],
GetItemsResourceValues['ItemInfo.Title'],
GetItemsResourceValues['ItemInfo.TradeInInfo'],
GetItemsResourceValues['Offers.Listings.Availability.MaxOrderQuantity'],
GetItemsResourceValues['Offers.Listings.Availability.Message'],
GetItemsResourceValues['Offers.Listings.Availability.MinOrderQuantity'],
GetItemsResourceValues['Offers.Listings.Availability.Type'],
GetItemsResourceValues['Offers.Listings.Condition'],
GetItemsResourceValues['Offers.Listings.Condition.ConditionNote'],
GetItemsResourceValues['Offers.Listings.Condition.SubCondition'],
GetItemsResourceValues['Offers.Listings.DeliveryInfo.IsAmazonFulfilled'],
GetItemsResourceValues['Offers.Listings.DeliveryInfo.IsFreeShippingEligible'],
GetItemsResourceValues['Offers.Listings.DeliveryInfo.IsPrimeEligible'],
GetItemsResourceValues['Offers.Listings.IsBuyBoxWinner'],
GetItemsResourceValues['Offers.Listings.LoyaltyPoints.Points'],
GetItemsResourceValues['Offers.Listings.MerchantInfo'],
GetItemsResourceValues['Offers.Listings.Price'],
GetItemsResourceValues['Offers.Listings.ProgramEligibility.IsPrimeExclusive'],
GetItemsResourceValues['Offers.Listings.ProgramEligibility.IsPrimePantry'],
GetItemsResourceValues['Offers.Listings.Promotions'],
GetItemsResourceValues['Offers.Listings.SavingBasis'],
GetItemsResourceValues['Offers.Summaries.HighestPrice'],
GetItemsResourceValues['Offers.Summaries.LowestPrice'],
GetItemsResourceValues['Offers.Summaries.OfferCount'],
GetItemsResourceValues['ParentASIN'],
GetItemsResourceValues['RentalOffers.Listings.Availability.MaxOrderQuantity'],
GetItemsResourceValues['RentalOffers.Listings.Availability.Message'],
GetItemsResourceValues['RentalOffers.Listings.Availability.MinOrderQuantity'],
GetItemsResourceValues['RentalOffers.Listings.Availability.Type'],
GetItemsResourceValues['RentalOffers.Listings.BasePrice'],
GetItemsResourceValues['RentalOffers.Listings.Condition'],
GetItemsResourceValues['RentalOffers.Listings.Condition.ConditionNote'],
GetItemsResourceValues['RentalOffers.Listings.Condition.SubCondition'],
GetItemsResourceValues['RentalOffers.Listings.DeliveryInfo.IsAmazonFulfilled'],
GetItemsResourceValues['RentalOffers.Listings.DeliveryInfo.IsFreeShippingEligible'],
GetItemsResourceValues['RentalOffers.Listings.DeliveryInfo.IsPrimeEligible'],
GetItemsResourceValues['RentalOffers.Listings.DeliveryInfo.ShippingCharges'],
GetItemsResourceValues['RentalOffers.Listings.MerchantInfo'],
];
return new Promise<ProductAdvertisingAPIv1.GetItemsResponse>((resolve, reject) => {
return api.getItems(getItemsRequest).then(
return api.getItems(ProductAdvertisingAPIv1.GetItemsRequest.constructFromObject(getItemsRequest)!).then(
function(data) {
console.log('API called successfully.');
const getItemsResponse = ProductAdvertisingAPIv1.GetItemsResponse.constructFromObject(data)!;

5
src/apps/amazon-catalog.ts
View File

@ -167,6 +167,7 @@ program.command('append-images')
})
.configureHelp();
// TODO: when I get back I need to build a command that lets me replay the procure-asins previous API request
program.command('procure-asins')
.alias('procure-asin')
@ -190,7 +191,7 @@ program.command('procure-asins')
}
const MAX_ITEMS_PER_API_REQUEST = 10;
for (let a = 0; a < asins.length; a += MAX_ITEMS_PER_API_REQUEST) {
let asinsToRequest = asins.slice(a, a + MAX_ITEMS_PER_API_REQUEST).map(asin => asin = asin.toLowerCase());
let asinsToRequest = asins.slice(a, a + MAX_ITEMS_PER_API_REQUEST).map(asin => asin = asin.toUpperCase());
console.log(`[procure-asin] Begin product enrollment(s) for ${asins.length} ASINs ${asinsToRequest.join(' ')}.`);
//get Amazon data ; we will use previously obtained data for the testing phase
@ -269,7 +270,7 @@ program.command('procure-asins')
console.log(`[procure-asin] Matching Brand ID: ${brandsDto.items[0].id}, Brand Name: ${brandsDto.items[0].data?.brandName['en-US']}.`);
}
let brandName = brandsDto.items[0].data?.brandName['en-US']!;
let features = amazonItem.ItemInfo?.Features?.DisplayValues!;
let features = amazonItem.ItemInfo?.Features?.DisplayValues!||[];
console.log(`[procure-asin] Amazon Product Name (en-US): ${amazonProductName_en_US}`);
console.log(`[procure-asin] Requesting llama3.1 LLM to translate product name from en-US to es-US.`);

8
src/apps/modules/amazon-catalog-helpers.ts
View File

@ -39,9 +39,11 @@ export async function lookupAmazonASINs(asins: string[]) {
// let apiResponse = response as GetItemsResponse;
let amazonGetItem: AmazonGetItem = {
apiResponse: { iv: apiResponse },
getItemsRequest: { iv: { schemaId: amazonGetItemsRequestSchemaDto.id, ItemIds: [ {
ItemId: (apiResponse.ItemsResult && apiResponse.ItemsResult.Items!.length > 0) ? apiResponse.ItemsResult!.Items![0].ASIN! : asins[0]
} ] } },
getItemsRequest: { iv: { schemaId: amazonGetItemsRequestSchemaDto.id, ItemIds: [
...((apiResponse.ItemsResult && apiResponse.ItemsResult.Items!.length > 0) ?
apiResponse.ItemsResult!.Items!.map((item) => { return { ItemId: item.ASIN! } }) :
asins.map((asin) => { return { ItemId: asin }}))
] } },
requestDate: { iv: requestDate }
}
let amazonGetItemDto = await client.contents.postContent(SCHEMAS.AMAZON_GET_ITEMS, {

16
src/pages/ads.txt.ts Normal file
View File

@ -0,0 +1,16 @@
import type { APIRoute } from 'astro';
import { config } from "../config";
import { getSiteConfig } from '../data/api-client';
const siteConfigDto = await getSiteConfig();
const siteConfig = siteConfigDto.items[0].data!;
const adsTxt = siteConfig.googleAdSense.iv.adsTxt;
export const GET: APIRoute = () => {
return new Response(adsTxt, {
headers: {
'Content-Type': 'text/plain; charset=utf-8',
},
});
};

23
src/pages/robots.txt.ts Normal file
View File

@ -0,0 +1,23 @@
import type { APIRoute } from 'astro';
const shouldPermitRobots = import.meta.env.MODE !== 'development';
const permissiveRobotsTxt = `
User-agent: *
Allow: /
Sitemap: ${new URL('sitemap-index.xml', import.meta.env.SITE).href}
`.trimStart();
const restrictiveRobotsTxt = `
User-agent: *
Disallow: /
`.trimStart();
export const GET: APIRoute = () => {
return new Response(shouldPermitRobots ? permissiveRobotsTxt : restrictiveRobotsTxt, {
headers: {
'Content-Type': 'text/plain; charset=utf-8',
},
});
};