diff --git a/package.json b/package.json index f981220..27197dd 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "dev": "astro dev", "start": "astro dev", "build": "astro check && astro build", + "catalog": "bun run src/apps/catalog/catalog.ts", "preview": "astro preview", "astro": "astro", "test": "vitest", diff --git a/src/apps/amazon-catalog.ts b/src/apps/amazon-catalog.ts deleted file mode 100644 index e2a4cd5..0000000 --- a/src/apps/amazon-catalog.ts +++ /dev/null @@ -1,404 +0,0 @@ -import { getBrandsByLangSlug, getBrandsUsingJsonQuery, getMarketplacesUsingJsonQuery, getProductCategoriesByIds, getProductCategoriesUsingJsonQuery, getProductListingsUsingJsonQuery, getProductsByIds, getProductsUsingJsonQuery, getSellersUsingJsonQuery, performSyncLocalizedSlugs } from '../data/api-client.ts'; -import { program } from 'commander'; -import * as core from '../data/core/client.js' -import type { AmazonMarketplaceConnection } from '../data/models/components/AmazonMarketplaceConnection.js'; -import type { Product } from '../data/models/multis/Product.js'; -import type { Listing } from '../data/models/multis/Listing.js'; -import type { Brand } from '../data/models/multis/Brand.js'; -import { askAiProductSubCategoryEvalQuestionsSet1, askAiProductSubCategoryEvalQuestionsSet2, askAiTopLevelProductCategoryEvalQuestions, designAiTopLevelProductCategoryEvalQuestions, doesProductAlreadyExist, generateAIProductDescription, generateAITagsForProduct, getAddNewBrandDtoByName, getAddNewProductDtoByProduct, getAddNewProductListingDtoByProduct, getAddNewProductSubCategoryDto, getAddNewSellerDtoByName, getAllTopLevelProductCategories, getAmazonMarketplaceConnectionSchemaDto, getAmazonMarketplaceDto, getBrandDtoByName, getMarketplaceConnectionSchemaDto, getSellerDtoByName, isValidASIN, removeQuotes, uploadDownloadedImageToSquidexAsAsset, translateAmazonDescription_from_en_US_to_es_US, translateAmazonDescription_from_en_US_to_fr_CA, translateProductDescription_from_en_US_to_es_US, translateProductName_from_en_US_to_es_US, translateProductName_from_en_US_to_fr_CA, translateTags_from_en_US_to_es_US, translateTags_from_en_US_to_fr_CA, trimPeriods, upsertAssetFolder, getAllAssetsInFolder, getAddNewOfferDto, lookupAmazonASINs, translateProductDescription_from_en_US_to_fr_CA } from './modules/amazon-catalog-helpers.js'; -import slugify from 'slugify'; -import type { Multilingual } from '../data/internals/MultilingualT.js'; -import type { NonMultilingual } from '../data/internals/NonMultilingualT.js'; -import type { MarketplaceConnection } from '../data/models/components/MarketplaceConnection.js'; -import type { Offer } from '../data/models/multis/Offer.js'; -import { SCHEMAS } from '../data/models/schemas.js'; -import { CheerioCrawler, type CheerioCrawlingContext, log } from 'crawlee'; -import { extractProductDetails } from '../scraper/amazon.js'; - -program.command('ls') - .description('List all the products') - .action(async (args: string[]) => { - let productsDto = await getProductsUsingJsonQuery(); - productsDto.items.forEach((productDto, index) => { - console.log(`[ls:${index+1}] ID: ${productDto.id}`) - console.log(`[ls:${index+1}] Product (en-US): ${productDto.data?.productName['en-US']}`); - console.log(`[ls:${index+1}] Product (es-US): ${productDto.data?.productName['es-US']}`); - console.log(`[ls:${index+1}] Product (fr-CA): ${productDto.data?.productName['fr-CA']}`); - let asin = productDto.data?.marketplaceConnections.iv.map((connection) => (connection.connection as AmazonMarketplaceConnection).asin).join(''); - let siteStripUrl = productDto.data?.marketplaceConnections.iv.map((connection) => (connection.connection as AmazonMarketplaceConnection).siteStripeUrl).join(''); - console.log(`[ls:${index+1}] ASIN: ${asin} ${!!(asin||'').match(/[A-Z0-9]{10}/g) ? 'Is a valid ASIN.' : 'Is not a valid ASIN.'}`) - console.log(`[ls:${index+1}] Amazon SiteStripe URL: ${siteStripUrl}`) - console.log(); - }); - console.log(`[ls] Returned ${productsDto.items.length} products.`) - }) - .configureHelp(); - -program.command('crawlee-products') - .alias('crawlee-product') - .argument('') - .description('Attempts to crawl Amazon product data using crawlee') - .action(async (urls: string[]) => { - /** - * Performs the logic of the crawler. It is called for each URL to crawl. - * - Passed to the crawler using the `requestHandler` option. - */ - const requestHandler = async (context: CheerioCrawlingContext) => { - const { $, request } = context; - const { url } = request; - log.info(`Scraping product page`, { url }); - const extractedProduct = extractProductDetails($); - log.info(`Scraped product details for "${extractedProduct.title}", saving...`, { url }); - crawler.pushData(extractedProduct); - }; - /** - * The crawler instance. Crawlee provides a few different crawlers, but we'll use CheerioCrawler, as it's very fast and simple to use. - * - Alternatively, we could use a full browser crawler like `PlaywrightCrawler` to imitate a real browser. - */ - const crawler = new CheerioCrawler({ requestHandler }); - await crawler.run(urls); - }) - .configureHelp(); - -program.command('sync-slugs') - .description('Sync URL slugs for each frontend endpoint.') - .action(async (asin: string, args: string[]) => { - await performSyncLocalizedSlugs(console.log); - }) - .configureHelp(); - -program.command('append-images') - .alias('append-image') - .argument('', 'Amazon Standard Identification Numbers') - .argument('', 'Image URLs to transmit to the database.') - .description('Download images from URL .') - .action(async (asin: string, urls: string[]) => { - if (!isValidASIN(asin)) { - console.error(`[append-images] error: ${asin} is not a valid ASIN. Amazon Standard Identification Numbers are 10-digits long with letters A-Z and numbers 0-9.`); - return; - } - let productsDto = await getProductsUsingJsonQuery(JSON.stringify({ filter: { - op: 'eq', - path: 'data.marketplaceConnections.iv.connection.asin', - value: asin, - }})); - if (productsDto.items.length === 0) { - console.error(`[append-images] error: ${asin} was not found in the database. Please procure or enter the product first.`); - return; - } - let marketplacesDto = await getMarketplacesUsingJsonQuery(JSON.stringify({ filter: { - op: 'eq', - path: 'data.marketplaceName.en-US', - value: 'Amazon', - }})); - if (marketplacesDto.items.length === 0) { - console.error(`[append-images] error: Amazon marketplace not found in database. Please set up Amazon marketplace in database.`); - return; - } - console.log(`[append-images] Upserting Asset Folder products`); - let productsAssetFolder = await upsertAssetFolder('products'); - console.log(`[append-images] Matching Asset Folder: ${productsAssetFolder.folderName} with Asset Folder ID: ${productsAssetFolder.id}`); - - console.log(`[append-images] Upserting Asset Folder ${productsAssetFolder.folderName}/amazon`); - let productsAmazonAssetFolder = await upsertAssetFolder('amazon', productsAssetFolder.id); - console.log(`[append-images] Matching Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName} with Asset Folder ID: ${productsAmazonAssetFolder.id}`); - - console.log(`[append-images] Upserting Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName}/${asin}`); - let productsASINFolder = await upsertAssetFolder(asin, productsAmazonAssetFolder.id); - console.log(`[append-images] Matching Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName}/${productsASINFolder.folderName} with Asset Folder ID: ${productsASINFolder.id}`); - let amazonMarketplaceDto = marketplacesDto.items[0]; - for (let productDto of productsDto.items) { - let listingsDto = await getProductListingsUsingJsonQuery(JSON.stringify({ filter: { - 'and': [ - { - op: 'eq', - path: 'data.product.iv', - value: productDto.id, - }, - { - op: 'eq', - path: 'data.marketplace.iv', - value: amazonMarketplaceDto.id, - } - ] - }})); - if (listingsDto.items.length === 0) { - console.error(`[append-images] error: Product listing for ${asin} on Amazon marketplace was not found in the database. Please procure or enter the product listing first.`); - return; - } - for (let listingDto of listingsDto.items) { - let listing = listingDto.data!; - let didUpdate = false; - - let amazonAssetsDto = await getAllAssetsInFolder(productsASINFolder.id); - - for (let i = 0; i < urls.length; i++) { - console.log(urls[i]); - let foundUploaded = amazonAssetsDto.filter((asset) => (asset.metadata['amazon-url'] as string||'') === urls[i]); - if (!foundUploaded.length) { // is not found - console.log(`[append-images] Transmitting Product Image ${urls[i]} to Squidex Assets`); - let assetDto = await uploadDownloadedImageToSquidexAsAsset(urls[i]!, productsASINFolder.id); - console.log(`[append-images] Saved Asset Id: ${assetDto.id} to Asset Folder Id: ${assetDto.parentId}`); - if (!listing.marketplaceImages) { - listing.marketplaceImages = { iv: [] }; - } - if (!listing.marketplaceImages.iv) { - listing.marketplaceImages.iv = []; - } - listing.marketplaceImages.iv.push(assetDto.id); - didUpdate = true; - } - else { // is found - console.log(`[append-images] Matched Asset Id: ${foundUploaded[0].id}, Amazon Product Image: ${foundUploaded[0].metadata['amazon-url'] as string||''}.`); - } - } - if (didUpdate) { - console.log(`[append-images] Listing did update, updating product listing with appended images.`); - let updatedDto = await core.client.contents.putContent(SCHEMAS.LISTINGS, listingDto.id, { - unpublished: false, - body: listing as any, - }, { - timeoutInSeconds: core.TIMEOUT_IN_SECONDS - }); - console.log(`[append-images] Listing version ${updatedDto.version} stored.`); - } - } - } - }) - .configureHelp(); - -// TODO: when I get back I need to build a command that lets me replay the procure-asins previous API request - -program.command('procure-asins') - .alias('procure-asin') - .argument('', 'Amazon Standard Identification Numbers') - .description('Begin automated product procurement from Amazon by inputting one or more ASINs separated by spaces.') - .action(async (asins: string[]) => { - if (!asins.length) { - console.error(`[procure-asin] error: You must specify one or more valid ASINs. Amazon Standard Identification Numbers are 10-digits long with letters A-Z and numbers 0-9.`); - return; - } - for (let a = 0; a < asins.length; a++) { - let asin = asins[a].toUpperCase(); - if (!isValidASIN(asin)) { - console.error(`[procure-asin] error: ${asin} is not a valid ASIN. Amazon Standard Identification Numbers are 10-digits long with letters A-Z and numbers 0-9.`); - return; - } - } - console.log(`[procure-asin] You started product enrollment for ${asins.length} items.`); - if (asins.length > 10) { - console.log(`[procure-asin] PA API calls will be broken up into 10 items per Amazon API request.`); - } - const MAX_ITEMS_PER_API_REQUEST = 10; - for (let a = 0; a < asins.length; a += MAX_ITEMS_PER_API_REQUEST) { - let asinsToRequest = asins.slice(a, a + MAX_ITEMS_PER_API_REQUEST).map(asin => asin = asin.toUpperCase()); - console.log(`[procure-asin] Begin product enrollment(s) for ${asins.length} ASINs ${asinsToRequest.join(' ')}.`); - - //get Amazon data ; we will use previously obtained data for the testing phase - //TODO:remove these three lines and replace with code that reads the Amazon PA API - // console.log(`[procure-asin] NOTICE: For testing purposes, using data from stored previous API response.`); - console.log(`[procure-asin] NOTICE: Production Amazon PA API request in progress.`); - const responseDto = await lookupAmazonASINs(asinsToRequest); - const apiResponse = responseDto.items[0].data?.apiResponse.iv!; - - apiResponse.ItemsResult!.Items!.forEach(async (amazonItem) => { - let asin = amazonItem.ASIN!; - console.log(`[procure-asin] Mock data override: product enrollment is for ASIN ${asin}.`); - - // enable this if we're saving data: - // if (await doesProductAlreadyExist(asin)) { - // console.error(`[procure-asin] error: Product with ASIN already exists in the application.`); - // return; - // } - - console.log(`[procure-asin] Amazon PA API Response contains ASIN: ${asin}.`); - const amazonProductName_en_US = amazonItem.ItemInfo?.Title?.DisplayValue!; - console.log(`[procure-asin] Amazon PA API Response contains Product Name: ${amazonProductName_en_US}.`); - console.log(`[procure-asin] Amazon PA API Response contains Product Features: \n${amazonItem.ItemInfo?.Features?.DisplayValues?.map((item) => `- ${item}`).join('\n')}`); - - const amazonProductImages = [amazonItem.Images?.Primary?.Large?.URL, ...(amazonItem.Images?.Variants?.map(variant => variant.Large?.URL)||[])]; - console.log(`[procure-asin] Amazon PA API Response contains Product Images: \n${amazonProductImages.map((url, index) => `${index+1}. ${url}`).join('\n')}`); - - console.log(`[procure-asin] Upserting Asset Folder products`); - let productsAssetFolder = await upsertAssetFolder('products'); - console.log(`[procure-asin] Matching Asset Folder: ${productsAssetFolder.folderName} with Asset Folder ID: ${productsAssetFolder.id}`); - - console.log(`[procure-asin] Upserting Asset Folder ${productsAssetFolder.folderName}/amazon`); - let productsAmazonAssetFolder = await upsertAssetFolder('amazon', productsAssetFolder.id); - console.log(`[procure-asin] Matching Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName} with Asset Folder ID: ${productsAmazonAssetFolder.id}`); - - console.log(`[procure-asin] Upserting Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName}/${asin}`); - let productsASINFolder = await upsertAssetFolder(asin, productsAmazonAssetFolder.id); - console.log(`[procure-asin] Matching Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName}/${productsASINFolder.folderName} with Asset Folder ID: ${productsASINFolder.id}`); - - let amazonAssetsDto = await getAllAssetsInFolder(productsASINFolder.id); - let amazonAssetsIds = amazonAssetsDto.map(asset => asset.id); - let amazonAssetsUrls: string[] = amazonAssetsDto.map(asset => asset.metadata['amazon-url'] as string||'').filter(url => url); - - for (let i = 0; i < amazonProductImages.length; i++) { - let foundUploaded = amazonAssetsDto.filter((asset) => (asset.metadata['amazon-url'] as string||'') === amazonProductImages[i]); - if (!foundUploaded.length) { // is not found - console.log(`[procure-asin] Transmitting Amazon Product Image ${amazonProductImages[i]} to Squidex Assets`); - let assetDto = await uploadDownloadedImageToSquidexAsAsset(amazonProductImages[i]!, productsASINFolder.id); - console.log(`[procure-asin] Saved Asset Id: ${assetDto.id} to Asset Folder Id: ${assetDto.parentId}`); - amazonAssetsDto.push(assetDto); - amazonAssetsIds.push(assetDto.id); - amazonAssetsUrls.push(assetDto.metadata['amazon-url'] as string||''); - } - else { // is found - console.log(`[procure-asin] Matched Asset Id: ${foundUploaded[0].id}, Amazon Product Image: ${foundUploaded[0].metadata['amazon-url'] as string||''}.`); - } - } - - let schemaProductMarketplaceConnectionDto = await getMarketplaceConnectionSchemaDto(); - console.log(`[procure-asin] Matching Schema ID: ${schemaProductMarketplaceConnectionDto.id}, Schema Name: ${schemaProductMarketplaceConnectionDto.name}.`); - - let schemaAmazonMarketplaceConnectionDto = await getAmazonMarketplaceConnectionSchemaDto(); - console.log(`[procure-asin] Matching Schema ID: ${schemaAmazonMarketplaceConnectionDto.id}, Schema Name: ${schemaAmazonMarketplaceConnectionDto.name}.`); - - let amazonMarketplaceDto = await getAmazonMarketplaceDto() - console.log(`[procure-asin] Matching Marketplace ID: ${amazonMarketplaceDto.items[0].id}, Marketplace Name: ${amazonMarketplaceDto.items[0].data?.marketplaceName['en-US']}.`); - - let bylineBrand = amazonItem.ItemInfo!.ByLineInfo!.Brand!.DisplayValue!; - console.log(`[procure-asin] Amazon PA API Response contains Brand ${bylineBrand}. Checking database for brand...`); - let brandsDto = await getBrandDtoByName(bylineBrand); - if (!brandsDto.items.length) { - console.log(`[procure-asin] Brand not found in database. Product requires creation of brand first.`); - brandsDto = await getAddNewBrandDtoByName(bylineBrand); - console.log(`[procure-asin] Brand Id: ${brandsDto.items[0].id}, Brand Name: ${brandsDto.items[0].data?.brandName['en-US']} created in database.`); - } else { - console.log(`[procure-asin] Matching Brand ID: ${brandsDto.items[0].id}, Brand Name: ${brandsDto.items[0].data?.brandName['en-US']}.`); - } - let brandName = brandsDto.items[0].data?.brandName['en-US']!; - let features = amazonItem.ItemInfo?.Features?.DisplayValues!||[]; - - console.log(`[procure-asin] Amazon Product Name (en-US): ${amazonProductName_en_US}`); - console.log(`[procure-asin] Requesting llama3.1 LLM to translate product name from en-US to es-US.`); - const amazonProductName_es_US = await translateProductName_from_en_US_to_es_US(amazonItem, brandName, amazonProductName_en_US); - console.log(`[procure-asin] AI Product Name (es-US): ${amazonProductName_es_US}`); - console.log(`[procure-asin] Requesting llama3.1 LLM to translate product name from en-US to fr-CA.`); - const amazonProductName_fr_CA = await translateProductName_from_en_US_to_fr_CA(amazonItem, brandName, amazonProductName_en_US); - console.log(`[procure-asin] AI Product Name (fr-CA): ${amazonProductName_fr_CA}`); - const aiProductDescriptionResponse_en_US = await generateAIProductDescription(amazonProductName_en_US, features); - console.log(`[procure-asin] Generated AI Product Description (en-US):\n${aiProductDescriptionResponse_en_US}`); - console.log(`[procure-asin] Requesting llama3.1 LLM to translate product description from en-US to es-US.`); - const aiProductDescriptionResponse_es_US = await translateProductDescription_from_en_US_to_es_US(brandName, aiProductDescriptionResponse_en_US); - console.log(`[procure-asin] Translated AI Product Description (es-US):\n${aiProductDescriptionResponse_es_US}`); - console.log(`[procure-asin] Requesting llama3.1 LLM to translate product description from en-US to fr-CA.`); - const aiProductDescriptionResponse_fr_CA = await translateProductDescription_from_en_US_to_fr_CA(brandName, aiProductDescriptionResponse_en_US); - console.log(`[procure-asin] AI Product Description (fr-CA):\n${aiProductDescriptionResponse_fr_CA}`); - const aiTopLevelCategoryChoice = await askAiTopLevelProductCategoryEvalQuestions(brandName, amazonProductName_en_US); - console.log('[procure-asin]', `AI chooses to put product within top level Product Category ID: ${aiTopLevelCategoryChoice}, Product Category Name: ${(await getAllTopLevelProductCategories()).filter((category) => category.id === aiTopLevelCategoryChoice)[0].categoryName!['en-US']}`); - const aiSubCategoryQuestion1Answer = await askAiProductSubCategoryEvalQuestionsSet1(aiTopLevelCategoryChoice, brandName, amazonProductName_en_US, features); - console.log('[procure-asin]', `AI then chooses within the top-level category to ${aiSubCategoryQuestion1Answer?'make a new sub-category':'use an existing sub-category'}.`); - const aiSubCategoryQuestion2Answer = await askAiProductSubCategoryEvalQuestionsSet2(apiResponse, aiTopLevelCategoryChoice, brandName, amazonProductName_en_US, features, aiSubCategoryQuestion1Answer);; - let aiProductCategoriesDto; - if (!aiSubCategoryQuestion1Answer) { - let aiSubCategoryQuestion2AnswerStr = aiSubCategoryQuestion2Answer as string; - console.log(`[procure-asin] AI suggested existing sub-category ${aiSubCategoryQuestion2Answer}`); - aiProductCategoriesDto = await getProductCategoriesByIds(aiSubCategoryQuestion2AnswerStr); - } - else { - let aiSubCategoryQuestion2AnswerObj = aiSubCategoryQuestion2Answer as { - categoryName: Multilingual, - description: Multilingual, - parentCategory: NonMultilingual, - }; - console.log(`[procure-asin] AI suggested new sub-category ${aiSubCategoryQuestion2AnswerObj.categoryName['en-US']}`); - aiProductCategoriesDto = await getAddNewProductSubCategoryDto({ iv: [aiTopLevelCategoryChoice] }, aiSubCategoryQuestion2AnswerObj.categoryName, aiSubCategoryQuestion2AnswerObj.description); - } - let aiProductTags_en_US = await generateAITagsForProduct(amazonProductName_en_US, features); - let aiProductTags_es_US = await translateTags_from_en_US_to_es_US(aiProductTags_en_US); - let aiProductTags_fr_CA = await translateTags_from_en_US_to_fr_CA(aiProductTags_en_US); - let product: Product = { - brand: { iv: [brandsDto.items![0].id!] }, - categories: { iv: aiProductCategoriesDto.items.map(category => category.id) }, - productName: { - "en-US": amazonProductName_en_US, - "es-US": amazonProductName_es_US, - "fr-CA": amazonProductName_fr_CA, - }, - slug: { - "en-US": `${brandsDto.items![0].data?.slug['en-US']}/${slugify(trimPeriods(removeQuotes(amazonProductName_en_US)), { lower: true, trim: true })}`, - "es-US": `${brandsDto.items![0].data?.slug['es-US']}/${slugify(trimPeriods(removeQuotes(amazonProductName_es_US)), { lower: true, trim: true })}`, - "fr-CA": `${brandsDto.items![0].data?.slug['fr-CA']}/${slugify(trimPeriods(removeQuotes(amazonProductName_fr_CA)), { lower: true, trim: true })}`, - }, - description: { - "en-US": aiProductDescriptionResponse_en_US, - "es-US": aiProductDescriptionResponse_es_US, - "fr-CA": aiProductDescriptionResponse_fr_CA, - }, - tags: { - "en-US": aiProductTags_en_US, - "es-US": aiProductTags_es_US, - "fr-CA": aiProductTags_fr_CA, - }, - marketplaceConnections: { - iv: [ - { - schemaId: schemaProductMarketplaceConnectionDto.id, - marketplace: [amazonMarketplaceDto.items[0].id], - connection: { - schemaId: schemaAmazonMarketplaceConnectionDto.id, - asin, - siteStripeUrl: amazonItem.DetailPageURL, - } as AmazonMarketplaceConnection | MarketplaceConnection - } - ] - } - }; - console.log(`[procure-asin] New product to store:`, product); - let productsDto = await getAddNewProductDtoByProduct(product); - console.log(`[procure-asin] Product Id: ${productsDto.items[0].id}, Product Name: ${productsDto.items[0].data?.productName['en-US']} created in database.`); - - let listing: Listing = { - marketplace: { iv: [ amazonMarketplaceDto.items[0].id ] }, - product: { iv: [ productsDto.items[0].id ]}, - marketplaceDescription: { - "en-US": `${features.map((feature) => `- ${feature}`).join('\n')}\n`, - "es-US": await translateAmazonDescription_from_en_US_to_es_US(brandName, features), - "fr-CA": await translateAmazonDescription_from_en_US_to_fr_CA(brandName, features), - }, - marketplaceImages: { iv: amazonAssetsIds }, - }; - console.log(`[procure-asin] New product listing to store:`, listing); - let listingsDto = await getAddNewProductListingDtoByProduct(listing); - console.log(`[procure-asin] Product Listing Id: ${listingsDto.items[0].id}, Product Id: ${listingsDto.items[0].data?.product.iv[0]}, Marketplace Id: ${listingsDto.items[0].data?.marketplace.iv[0]} created in database.`); - - amazonItem.Offers?.Listings?.forEach(async (amazonListing) => { - console.log(`[procure-asin] Amazon PA API Response contains Seller ${amazonListing.MerchantInfo?.Name} offer for ${amazonListing.Price?.Amount} in ${amazonListing.Condition?.Value} condition. Checking database for seller...`); - let sellersDto = await getSellerDtoByName(amazonListing.MerchantInfo!.Name!); - if (!sellersDto.items.length) { - console.log(`[procure-asin] Seller not found in database. Listing requires creation of seller first.`); - sellersDto = await getAddNewSellerDtoByName(amazonListing.MerchantInfo!.Name!); - console.log(`[procure-asin] Seller Id: ${sellersDto.items[0].id}, Seller Name: ${sellersDto.items[0].data?.sellerName['en-US']} created in database.`); - } else { - console.log(`[procure-asin] Matching Seller ID: ${sellersDto.items[0].id}, Seller Name: ${sellersDto.items[0].data?.sellerName['en-US']}.`); - } - let offer: Offer = { - offerDate: { iv: new Date().toISOString() }, - listing: { iv: [ listingsDto.items[0].id ] }, - seller: { iv: [ sellersDto.items[0].id ] }, - newPrice: { iv: null }, - usedPrice: { iv: null }, - }; - if (amazonListing.Condition?.Value === 'New') { - offer.newPrice = { iv: amazonListing.Price?.Amount!}; - offer.usedPrice = { iv: null }; - } - else if (amazonListing.Condition?.Value === 'Used') { - offer.newPrice = { iv: null }; - offer.usedPrice = { iv: amazonListing.Price?.Amount! } - } - console.log('Generated Offer:\n', offer); - let offersDto = await getAddNewOfferDto(offer); - }); - - await performSyncLocalizedSlugs(console.log); - }); - } - }) - .configureHelp(); - - -program.parse(); diff --git a/src/apps/catalog/amazon/amazon-append-images.ts b/src/apps/catalog/amazon/amazon-append-images.ts new file mode 100644 index 0000000..df31505 --- /dev/null +++ b/src/apps/catalog/amazon/amazon-append-images.ts @@ -0,0 +1,111 @@ +import type { Command } from "commander"; +import * as core from '../../../data/core/client' +import { getAllAssetsInFolder, isValidASIN, uploadDownloadedImageToSquidexAsAsset, upsertAssetFolder } from "../common/catalog-helpers"; +import { getMarketplacesUsingJsonQuery, getProductListingsUsingJsonQuery, getProductsUsingJsonQuery } from "../../../data/api-client"; +import { SCHEMAS } from "../../../data/models/schemas"; +import { logForCommand } from "../common/console"; + +export const COMMAND_NAME = 'append-images'; + +const log = logForCommand(COMMAND_NAME); +const error = logForCommand(COMMAND_NAME, console.error); + +export const amazonAppendImagesCommand = (amazonCommand: Command) => + amazonCommand.command(COMMAND_NAME).alias('append-image') + .description('Download images from URL .') + .argument('', 'Amazon Standard Identification Numbers') + .argument('', 'Image URLs to transmit to the database.') + .action(async (asin: string, urls: string[]) => { + if (!isValidASIN(asin)) { + error(`error: ${asin} is not a valid ASIN. Amazon Standard Identification Numbers are 10-digits long with letters A-Z and numbers 0-9.`); + return; + } + let productsDto = await getProductsUsingJsonQuery(JSON.stringify({ filter: { + op: 'eq', + path: 'data.marketplaceConnections.iv.connection.asin', + value: asin, + }})); + if (productsDto.items.length === 0) { + error(`error: ${asin} was not found in the database. Please procure or enter the product first.`); + return; + } + let marketplacesDto = await getMarketplacesUsingJsonQuery(JSON.stringify({ filter: { + op: 'eq', + path: 'data.marketplaceName.en-US', + value: 'Amazon', + }})); + if (marketplacesDto.items.length === 0) { + error(`error: Amazon marketplace not found in database. Please set up Amazon marketplace in database.`); + return; + } + log(`Upserting Asset Folder products`); + let productsAssetFolder = await upsertAssetFolder('products'); + log(`Matching Asset Folder: ${productsAssetFolder.folderName} with Asset Folder ID: ${productsAssetFolder.id}`); + + log(`Upserting Asset Folder ${productsAssetFolder.folderName}/amazon`); + let productsAmazonAssetFolder = await upsertAssetFolder('amazon', productsAssetFolder.id); + log(`Matching Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName} with Asset Folder ID: ${productsAmazonAssetFolder.id}`); + + log(`Upserting Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName}/${asin}`); + let productsASINFolder = await upsertAssetFolder(asin, productsAmazonAssetFolder.id); + log(`Matching Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName}/${productsASINFolder.folderName} with Asset Folder ID: ${productsASINFolder.id}`); + let amazonMarketplaceDto = marketplacesDto.items[0]; + for (let productDto of productsDto.items) { + let listingsDto = await getProductListingsUsingJsonQuery(JSON.stringify({ filter: { + 'and': [ + { + op: 'eq', + path: 'data.product.iv', + value: productDto.id, + }, + { + op: 'eq', + path: 'data.marketplace.iv', + value: amazonMarketplaceDto.id, + } + ] + }})); + if (listingsDto.items.length === 0) { + error(`error: Product listing for ${asin} on Amazon marketplace was not found in the database. Please procure or enter the product listing first.`); + return; + } + for (let listingDto of listingsDto.items) { + let listing = listingDto.data!; + let didUpdate = false; + + let amazonAssetsDto = await getAllAssetsInFolder(productsASINFolder.id); + + for (let i = 0; i < urls.length; i++) { + log(urls[i]); + let foundUploaded = amazonAssetsDto.filter((asset) => (asset.metadata['amazon-url'] as string||'') === urls[i]); + if (!foundUploaded.length) { // is not found + log(`Transmitting Product Image ${urls[i]} to Squidex Assets`); + let assetDto = await uploadDownloadedImageToSquidexAsAsset(urls[i]!, productsASINFolder.id); + log(`Saved Asset Id: ${assetDto.id} to Asset Folder Id: ${assetDto.parentId}`); + if (!listing.marketplaceImages) { + listing.marketplaceImages = { iv: [] }; + } + if (!listing.marketplaceImages.iv) { + listing.marketplaceImages.iv = []; + } + listing.marketplaceImages.iv.push(assetDto.id); + didUpdate = true; + } + else { // is found + log(`Matched Asset Id: ${foundUploaded[0].id}, Amazon Product Image: ${foundUploaded[0].metadata['amazon-url'] as string||''}.`); + } + } + if (didUpdate) { + log(`Listing did update, updating product listing with appended images.`); + let updatedDto = await core.client.contents.putContent(SCHEMAS.LISTINGS, listingDto.id, { + unpublished: false, + body: listing as any, + }, { + timeoutInSeconds: core.TIMEOUT_IN_SECONDS + }); + log(`Listing version ${updatedDto.version} stored.`); + } + } + } + }) + .configureHelp(); \ No newline at end of file diff --git a/src/apps/catalog/amazon/amazon-crawlee-products.ts b/src/apps/catalog/amazon/amazon-crawlee-products.ts new file mode 100644 index 0000000..9d28ef9 --- /dev/null +++ b/src/apps/catalog/amazon/amazon-crawlee-products.ts @@ -0,0 +1,34 @@ +import type { Command } from "commander"; +import { CheerioCrawler, log as crawleeLog, type CheerioCrawlingContext } from "crawlee"; +import { extractProductDetails } from "../../../scraper/amazon"; +import { logForCommand } from "../common/console"; + +export const COMMAND_NAME = 'crawlee-products'; + +const log = logForCommand(COMMAND_NAME, crawleeLog.info); + +export const amazonCrawleeProductsCommand = (amazonCommand: Command) => + amazonCommand.command(COMMAND_NAME).alias('crawlee-product') + .description('Attempts to crawl Amazon product data using crawlee') + .argument('') + .action(async (urls: string[]) => { + /** + * Performs the logic of the crawler. It is called for each URL to crawl. + * - Passed to the crawler using the `requestHandler` option. + */ + const requestHandler = async (context: CheerioCrawlingContext) => { + const { $, request } = context; + const { url } = request; + log(`Scraping product page`, { url }); + const extractedProduct = extractProductDetails($); + log(`Scraped product details for "${extractedProduct.title}", saving...`, { url }); + crawler.pushData(extractedProduct); + }; + /** + * The crawler instance. Crawlee provides a few different crawlers, but we'll use CheerioCrawler, as it's very fast and simple to use. + * - Alternatively, we could use a full browser crawler like `PlaywrightCrawler` to imitate a real browser. + */ + const crawler = new CheerioCrawler({ requestHandler }); + await crawler.run(urls); + }) + .configureHelp(); diff --git a/src/apps/catalog/amazon/amazon-procure-asins.ts b/src/apps/catalog/amazon/amazon-procure-asins.ts new file mode 100644 index 0000000..f71a61b --- /dev/null +++ b/src/apps/catalog/amazon/amazon-procure-asins.ts @@ -0,0 +1,251 @@ +import type { Command } from "commander"; +import { askAiProductSubCategoryEvalQuestionsSet1, askAiProductSubCategoryEvalQuestionsSet2, askAiTopLevelProductCategoryEvalQuestions, generateAIProductDescription, generateAITagsForProduct, getAddNewBrandDtoByName, getAddNewOfferDto, getAddNewProductDtoByProduct, getAddNewProductListingDtoByProduct, getAddNewProductSubCategoryDto, getAddNewSellerDtoByName, getAllAssetsInFolder, getAllTopLevelProductCategories, getAmazonMarketplaceConnectionSchemaDto, getAmazonMarketplaceDto, getBrandDtoByName, getMarketplaceConnectionSchemaDto, getSellerDtoByName, isValidASIN, lookupAmazonASINs, removeQuotes, translateAmazonDescription_from_en_US_to_es_US, translateAmazonDescription_from_en_US_to_fr_CA, translateProductDescription_from_en_US_to_es_US, translateProductDescription_from_en_US_to_fr_CA, translateProductName_from_en_US_to_es_US, translateProductName_from_en_US_to_fr_CA, translateTags_from_en_US_to_es_US, translateTags_from_en_US_to_fr_CA, trimPeriods, uploadDownloadedImageToSquidexAsAsset, upsertAssetFolder } from "../common/catalog-helpers"; +import type { Localized } from "../../../data/internals/LocalizedT"; +import type { NonLocalized } from "../../../data/internals/NonLocalizedT"; +import { getProductCategoriesByIds, performSyncLocalizedSlugs } from "../../../data/api-client"; +import type { Product } from "../../../data/models/multis/Product"; +import slugify from "slugify"; +import type { AmazonMarketplaceConnection } from "../../../data/models/components/AmazonMarketplaceConnection"; +import type { MarketplaceConnection } from "../../../data/models/components/MarketplaceConnection"; +import type { Listing } from "../../../data/models/multis/Listing"; +import type { Offer } from "../../../data/models/multis/Offer"; +import { logForCommand } from "../common/console"; + +export const COMMAND_NAME = 'procure-asins'; + +const log = logForCommand(COMMAND_NAME); +const error = logForCommand(COMMAND_NAME, console.error); + +export const amazonProcureASINsCommand = (amazonCommand: Command) => { + let command = amazonCommand.command(COMMAND_NAME) + .alias('procure-asin') + .argument('', 'Amazon Standard Identification Numbers') + .description('Begin automated product procurement from Amazon by inputting one or more ASINs separated by spaces.') + .action(async (asins: string[]) => { + if (!asins.length) { + error(`error: You must specify one or more valid ASINs. Amazon Standard Identification Numbers are 10-digits long with letters A-Z and numbers 0-9.`); + return; + } + for (let a = 0; a < asins.length; a++) { + let asin = asins[a].toUpperCase(); + if (!isValidASIN(asin)) { + error(`error: ${asin} is not a valid ASIN. Amazon Standard Identification Numbers are 10-digits long with letters A-Z and numbers 0-9.`); + return; + } + } + log(`You started product enrollment for ${asins.length} items.`); + if (asins.length > 10) { + log(`PA API calls will be broken up into 10 items per Amazon API request.`); + } + const MAX_ITEMS_PER_API_REQUEST = 10; + for (let a = 0; a < asins.length; a += MAX_ITEMS_PER_API_REQUEST) { + let asinsToRequest = asins.slice(a, a + MAX_ITEMS_PER_API_REQUEST).map(asin => asin = asin.toUpperCase()); + log(`Begin product enrollment(s) for ${asins.length} ASINs ${asinsToRequest.join(' ')}.`); + + //get Amazon data ; we will use previously obtained data for the testing phase + //TODO:remove these three lines and replace with code that reads the Amazon PA API + // log(`NOTICE: For testing purposes, using data from stored previous API response.`); + log(`NOTICE: Production Amazon PA API request in progress.`); + const responseDto = await lookupAmazonASINs(asinsToRequest); + const apiResponse = responseDto.items[0].data?.apiResponse.iv!; + + apiResponse.ItemsResult!.Items!.forEach(async (amazonItem) => { + let asin = amazonItem.ASIN!; + log(`Mock data override: product enrollment is for ASIN ${asin}.`); + + // enable this if we're saving data: + // if (await doesProductAlreadyExist(asin)) { + // error(`error: Product with ASIN already exists in the application.`); + // return; + // } + + log(`Amazon PA API Response contains ASIN: ${asin}.`); + const amazonProductName_en_US = amazonItem.ItemInfo?.Title?.DisplayValue!; + log(`Amazon PA API Response contains Product Name: ${amazonProductName_en_US}.`); + log(`Amazon PA API Response contains Product Features: \n${amazonItem.ItemInfo?.Features?.DisplayValues?.map((item) => `- ${item}`).join('\n')}`); + + const amazonProductImages = [amazonItem.Images?.Primary?.Large?.URL, ...(amazonItem.Images?.Variants?.map(variant => variant.Large?.URL)||[])]; + log(`Amazon PA API Response contains Product Images: \n${amazonProductImages.map((url, index) => `${index+1}. ${url}`).join('\n')}`); + + log(`Upserting Asset Folder products`); + let productsAssetFolder = await upsertAssetFolder('products'); + log(`Matching Asset Folder: ${productsAssetFolder.folderName} with Asset Folder ID: ${productsAssetFolder.id}`); + + log(`Upserting Asset Folder ${productsAssetFolder.folderName}/amazon`); + let productsAmazonAssetFolder = await upsertAssetFolder('amazon', productsAssetFolder.id); + log(`Matching Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName} with Asset Folder ID: ${productsAmazonAssetFolder.id}`); + + log(`Upserting Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName}/${asin}`); + let productsASINFolder = await upsertAssetFolder(asin, productsAmazonAssetFolder.id); + log(`Matching Asset Folder ${productsAssetFolder.folderName}/${productsAmazonAssetFolder.folderName}/${productsASINFolder.folderName} with Asset Folder ID: ${productsASINFolder.id}`); + + let amazonAssetsDto = await getAllAssetsInFolder(productsASINFolder.id); + let amazonAssetsIds = amazonAssetsDto.map(asset => asset.id); + let amazonAssetsUrls: string[] = amazonAssetsDto.map(asset => asset.metadata['amazon-url'] as string||'').filter(url => url); + + for (let i = 0; i < amazonProductImages.length; i++) { + let foundUploaded = amazonAssetsDto.filter((asset) => (asset.metadata['amazon-url'] as string||'') === amazonProductImages[i]); + if (!foundUploaded.length) { // is not found + log(`Transmitting Amazon Product Image ${amazonProductImages[i]} to Squidex Assets`); + let assetDto = await uploadDownloadedImageToSquidexAsAsset(amazonProductImages[i]!, productsASINFolder.id); + log(`Saved Asset Id: ${assetDto.id} to Asset Folder Id: ${assetDto.parentId}`); + amazonAssetsDto.push(assetDto); + amazonAssetsIds.push(assetDto.id); + amazonAssetsUrls.push(assetDto.metadata['amazon-url'] as string||''); + } + else { // is found + log(`Matched Asset Id: ${foundUploaded[0].id}, Amazon Product Image: ${foundUploaded[0].metadata['amazon-url'] as string||''}.`); + } + } + + let schemaProductMarketplaceConnectionDto = await getMarketplaceConnectionSchemaDto(); + log(`Matching Schema ID: ${schemaProductMarketplaceConnectionDto.id}, Schema Name: ${schemaProductMarketplaceConnectionDto.name}.`); + + let schemaAmazonMarketplaceConnectionDto = await getAmazonMarketplaceConnectionSchemaDto(); + log(`Matching Schema ID: ${schemaAmazonMarketplaceConnectionDto.id}, Schema Name: ${schemaAmazonMarketplaceConnectionDto.name}.`); + + let amazonMarketplaceDto = await getAmazonMarketplaceDto() + log(`Matching Marketplace ID: ${amazonMarketplaceDto.items[0].id}, Marketplace Name: ${amazonMarketplaceDto.items[0].data?.marketplaceName['en-US']}.`); + + let bylineBrand = amazonItem.ItemInfo!.ByLineInfo!.Brand!.DisplayValue!; + log(`Amazon PA API Response contains Brand ${bylineBrand}. Checking database for brand...`); + let brandsDto = await getBrandDtoByName(bylineBrand); + if (!brandsDto.items.length) { + log(`Brand not found in database. Product requires creation of brand first.`); + brandsDto = await getAddNewBrandDtoByName(bylineBrand); + log(`Brand Id: ${brandsDto.items[0].id}, Brand Name: ${brandsDto.items[0].data?.brandName['en-US']} created in database.`); + } else { + log(`Matching Brand ID: ${brandsDto.items[0].id}, Brand Name: ${brandsDto.items[0].data?.brandName['en-US']}.`); + } + let brandName = brandsDto.items[0].data?.brandName['en-US']!; + let features = amazonItem.ItemInfo?.Features?.DisplayValues!||[]; + + log(`Amazon Product Name (en-US): ${amazonProductName_en_US}`); + log(`Requesting llama3.1 LLM to translate product name from en-US to es-US.`); + const amazonProductName_es_US = await translateProductName_from_en_US_to_es_US(amazonItem, brandName, amazonProductName_en_US); + log(`AI Product Name (es-US): ${amazonProductName_es_US}`); + log(`Requesting llama3.1 LLM to translate product name from en-US to fr-CA.`); + const amazonProductName_fr_CA = await translateProductName_from_en_US_to_fr_CA(amazonItem, brandName, amazonProductName_en_US); + log(`AI Product Name (fr-CA): ${amazonProductName_fr_CA}`); + const aiProductDescriptionResponse_en_US = await generateAIProductDescription(amazonProductName_en_US, features); + log(`Generated AI Product Description (en-US):\n${aiProductDescriptionResponse_en_US}`); + log(`Requesting llama3.1 LLM to translate product description from en-US to es-US.`); + const aiProductDescriptionResponse_es_US = await translateProductDescription_from_en_US_to_es_US(brandName, aiProductDescriptionResponse_en_US); + log(`Translated AI Product Description (es-US):\n${aiProductDescriptionResponse_es_US}`); + log(`Requesting llama3.1 LLM to translate product description from en-US to fr-CA.`); + const aiProductDescriptionResponse_fr_CA = await translateProductDescription_from_en_US_to_fr_CA(brandName, aiProductDescriptionResponse_en_US); + log(`AI Product Description (fr-CA):\n${aiProductDescriptionResponse_fr_CA}`); + const aiTopLevelCategoryChoice = await askAiTopLevelProductCategoryEvalQuestions(brandName, amazonProductName_en_US); + log('[procure-asin]', `AI chooses to put product within top level Product Category ID: ${aiTopLevelCategoryChoice}, Product Category Name: ${(await getAllTopLevelProductCategories()).filter((category) => category.id === aiTopLevelCategoryChoice)[0].categoryName!['en-US']}`); + const aiSubCategoryQuestion1Answer = await askAiProductSubCategoryEvalQuestionsSet1(aiTopLevelCategoryChoice, brandName, amazonProductName_en_US, features); + log('[procure-asin]', `AI then chooses within the top-level category to ${aiSubCategoryQuestion1Answer?'make a new sub-category':'use an existing sub-category'}.`); + const aiSubCategoryQuestion2Answer = await askAiProductSubCategoryEvalQuestionsSet2(apiResponse, aiTopLevelCategoryChoice, brandName, amazonProductName_en_US, features, aiSubCategoryQuestion1Answer);; + let aiProductCategoriesDto; + if (!aiSubCategoryQuestion1Answer) { + let aiSubCategoryQuestion2AnswerStr = aiSubCategoryQuestion2Answer as string; + log(`AI suggested existing sub-category ${aiSubCategoryQuestion2Answer}`); + aiProductCategoriesDto = await getProductCategoriesByIds(aiSubCategoryQuestion2AnswerStr); + } + else { + let aiSubCategoryQuestion2AnswerObj = aiSubCategoryQuestion2Answer as { + categoryName: Localized, + description: Localized, + parentCategory: NonLocalized, + }; + log(`AI suggested new sub-category ${aiSubCategoryQuestion2AnswerObj.categoryName['en-US']}`); + aiProductCategoriesDto = await getAddNewProductSubCategoryDto({ iv: [aiTopLevelCategoryChoice] }, aiSubCategoryQuestion2AnswerObj.categoryName, aiSubCategoryQuestion2AnswerObj.description); + } + let aiProductTags_en_US = await generateAITagsForProduct(amazonProductName_en_US, features); + let aiProductTags_es_US = await translateTags_from_en_US_to_es_US(aiProductTags_en_US); + let aiProductTags_fr_CA = await translateTags_from_en_US_to_fr_CA(aiProductTags_en_US); + let product: Product = { + brand: { iv: [brandsDto.items![0].id!] }, + categories: { iv: aiProductCategoriesDto.items.map(category => category.id) }, + productName: { + "en-US": amazonProductName_en_US, + "es-US": amazonProductName_es_US, + "fr-CA": amazonProductName_fr_CA, + }, + slug: { + "en-US": `${brandsDto.items![0].data?.slug['en-US']}/${slugify(trimPeriods(removeQuotes(amazonProductName_en_US)), { lower: true, trim: true })}`, + "es-US": `${brandsDto.items![0].data?.slug['es-US']}/${slugify(trimPeriods(removeQuotes(amazonProductName_es_US)), { lower: true, trim: true })}`, + "fr-CA": `${brandsDto.items![0].data?.slug['fr-CA']}/${slugify(trimPeriods(removeQuotes(amazonProductName_fr_CA)), { lower: true, trim: true })}`, + }, + description: { + "en-US": aiProductDescriptionResponse_en_US, + "es-US": aiProductDescriptionResponse_es_US, + "fr-CA": aiProductDescriptionResponse_fr_CA, + }, + tags: { + "en-US": aiProductTags_en_US, + "es-US": aiProductTags_es_US, + "fr-CA": aiProductTags_fr_CA, + }, + marketplaceConnections: { + iv: [ + { + schemaId: schemaProductMarketplaceConnectionDto.id, + marketplace: [amazonMarketplaceDto.items[0].id], + connection: { + schemaId: schemaAmazonMarketplaceConnectionDto.id, + asin, + siteStripeUrl: amazonItem.DetailPageURL, + } as AmazonMarketplaceConnection | MarketplaceConnection + } + ] + } + }; + log(`New product to store:`, product); + let productsDto = await getAddNewProductDtoByProduct(product); + log(`Product Id: ${productsDto.items[0].id}, Product Name: ${productsDto.items[0].data?.productName['en-US']} created in database.`); + + let listing: Listing = { + marketplace: { iv: [ amazonMarketplaceDto.items[0].id ] }, + product: { iv: [ productsDto.items[0].id ]}, + marketplaceDescription: { + "en-US": `${features.map((feature) => `- ${feature}`).join('\n')}\n`, + "es-US": await translateAmazonDescription_from_en_US_to_es_US(brandName, features), + "fr-CA": await translateAmazonDescription_from_en_US_to_fr_CA(brandName, features), + }, + marketplaceImages: { iv: amazonAssetsIds }, + }; + log(`New product listing to store:`, listing); + let listingsDto = await getAddNewProductListingDtoByProduct(listing); + log(`Product Listing Id: ${listingsDto.items[0].id}, Product Id: ${listingsDto.items[0].data?.product.iv[0]}, Marketplace Id: ${listingsDto.items[0].data?.marketplace.iv[0]} created in database.`); + + amazonItem.Offers?.Listings?.forEach(async (amazonListing) => { + log(`Amazon PA API Response contains Seller ${amazonListing.MerchantInfo?.Name} offer for ${amazonListing.Price?.Amount} in ${amazonListing.Condition?.Value} condition. Checking database for seller...`); + let sellersDto = await getSellerDtoByName(amazonListing.MerchantInfo!.Name!); + if (!sellersDto.items.length) { + log(`Seller not found in database. Listing requires creation of seller first.`); + sellersDto = await getAddNewSellerDtoByName(amazonListing.MerchantInfo!.Name!); + log(`Seller Id: ${sellersDto.items[0].id}, Seller Name: ${sellersDto.items[0].data?.sellerName['en-US']} created in database.`); + } else { + log(`Matching Seller ID: ${sellersDto.items[0].id}, Seller Name: ${sellersDto.items[0].data?.sellerName['en-US']}.`); + } + let offer: Offer = { + offerDate: { iv: new Date().toISOString() }, + listing: { iv: [ listingsDto.items[0].id ] }, + seller: { iv: [ sellersDto.items[0].id ] }, + newPrice: { iv: null }, + usedPrice: { iv: null }, + }; + if (amazonListing.Condition?.Value === 'New') { + offer.newPrice = { iv: amazonListing.Price?.Amount!}; + offer.usedPrice = { iv: null }; + } + else if (amazonListing.Condition?.Value === 'Used') { + offer.newPrice = { iv: null }; + offer.usedPrice = { iv: amazonListing.Price?.Amount! } + } + log('Generated Offer:\n', offer); + let offersDto = await getAddNewOfferDto(offer); + }); + + await performSyncLocalizedSlugs(log); + }); + } + }); + command.configureHelp(); + return command; +} \ No newline at end of file diff --git a/src/apps/catalog/amazon/amazon.ts b/src/apps/catalog/amazon/amazon.ts new file mode 100644 index 0000000..6cfdbb9 --- /dev/null +++ b/src/apps/catalog/amazon/amazon.ts @@ -0,0 +1,17 @@ +import type { Command } from "commander"; +import { amazonAppendImagesCommand } from "./amazon-append-images"; +import { amazonCrawleeProductsCommand } from "./amazon-crawlee-products"; +import { amazonProcureASINsCommand } from "./amazon-procure-asins"; +import { amazonGetItemsCommand } from "./get-items/get-items"; + +const COMMAND_NAME = 'amazon'; + +export const amazonCommand = (program: Command) => { + const amazonCommand = program.command(COMMAND_NAME).description('Amazon integration commands'); + amazonAppendImagesCommand(amazonCommand); + amazonCrawleeProductsCommand(amazonCommand); + amazonGetItemsCommand(amazonCommand); + amazonProcureASINsCommand(amazonCommand); + // TODO: when I get back I need to build a command that lets me replay the procure-asins previous API request + return amazonCommand; +} \ No newline at end of file diff --git a/src/apps/catalog/amazon/get-items/amazon-get-items-logs.ts b/src/apps/catalog/amazon/get-items/amazon-get-items-logs.ts new file mode 100644 index 0000000..1f6b864 --- /dev/null +++ b/src/apps/catalog/amazon/get-items/amazon-get-items-logs.ts @@ -0,0 +1,28 @@ +import type { Command } from "commander"; +import * as core from '../../../../data/core/client' +import { getProductsUsingJsonQuery } from "../../../../data/api-client"; +import type { AmazonMarketplaceConnection } from "../../../../data/models/components/AmazonMarketplaceConnection"; +import { logForCommand } from "../../common/console"; +import { SCHEMAS } from "../../../../data/models/schemas"; +import type { AmazonGetItem } from "../../../../data/models/multis/AmazonGetItem"; +import { DateTime } from "luxon"; + +export const COMMAND_NAME = 'logs'; + +const log = console.log; + +export const amazonGetItemsLogsCommand = (amazonGetItemsCommand: Command) => { + const command = amazonGetItemsCommand.command(COMMAND_NAME).alias('log') + .description('Prints the get-items JSON response') + .argument('') + .action(async (logIds: string[]) => { + for (let logId of logIds) { + let logDto = await core.getContentsByIds(SCHEMAS.AMAZON_GET_ITEMS, logId); + logDto.items.forEach((logDto, index) => { + log(JSON.stringify(logDto.data?.apiResponse, null, 2)); + }); + } + }); + command.configureHelp(); + return command; +} \ No newline at end of file diff --git a/src/apps/catalog/amazon/get-items/amazon-get-items-ls.ts b/src/apps/catalog/amazon/get-items/amazon-get-items-ls.ts new file mode 100644 index 0000000..ba30020 --- /dev/null +++ b/src/apps/catalog/amazon/get-items/amazon-get-items-ls.ts @@ -0,0 +1,27 @@ +import type { Command } from "commander"; +import * as core from '../../../../data/core/client' +import { getProductsUsingJsonQuery } from "../../../../data/api-client"; +import type { AmazonMarketplaceConnection } from "../../../../data/models/components/AmazonMarketplaceConnection"; +import { logForCommand } from "../../common/console"; +import { SCHEMAS } from "../../../../data/models/schemas"; +import type { AmazonGetItem } from "../../../../data/models/multis/AmazonGetItem"; +import { DateTime } from "luxon"; + +export const COMMAND_NAME = 'ls'; + +const log = logForCommand(COMMAND_NAME); + +export const amazonGetItemsLsCommand = (amazonGetItemsCommand: Command) => { + const command = amazonGetItemsCommand.command(COMMAND_NAME).alias('list') + .description('List all the get-items logs') + .action(async (args: string[]) => { + let logsDto = await core.getContentsUsingJsonQuery(SCHEMAS.AMAZON_GET_ITEMS); + logsDto.items.forEach((logDto, index) => { + const logEach = logForCommand(`${COMMAND_NAME}:${index+1}`); + logEach(`ID: ${logDto.id} Request Date: ${DateTime.fromISO(logDto.data?.requestDate.iv!).toLocaleString(DateTime.DATETIME_SHORT_WITH_SECONDS)}`) + }); + log(`Returned ${logsDto.items.length} get-items logs.`) + }); + command.configureHelp(); + return command; +} \ No newline at end of file diff --git a/src/apps/catalog/amazon/get-items/get-items.ts b/src/apps/catalog/amazon/get-items/get-items.ts new file mode 100644 index 0000000..d5ecece --- /dev/null +++ b/src/apps/catalog/amazon/get-items/get-items.ts @@ -0,0 +1,12 @@ +import type { Command } from "commander"; +import { amazonGetItemsLsCommand } from "./amazon-get-items-ls"; +import { amazonGetItemsLogsCommand } from "./amazon-get-items-logs"; + +const COMMAND_NAME = 'get-items'; + +export const amazonGetItemsCommand = (program: Command) => { + const amazonGetItemsCommand = program.command(COMMAND_NAME).alias('get-item').description('Amazon Get Items commands'); + amazonGetItemsLsCommand(amazonGetItemsCommand); + amazonGetItemsLogsCommand(amazonGetItemsCommand); + return amazonGetItemsCommand; +} \ No newline at end of file diff --git a/src/apps/catalog/catalog.ts b/src/apps/catalog/catalog.ts new file mode 100644 index 0000000..456c63a --- /dev/null +++ b/src/apps/catalog/catalog.ts @@ -0,0 +1,10 @@ +import { program } from 'commander'; +import { amazonCommand } from './amazon/amazon'; +import { productsCommand } from './products/products'; +import { siteCommands } from './site/site'; + +const amazon = amazonCommand(program); +const products = productsCommand(program); +const site = siteCommands(program); + +program.parse(); diff --git a/src/apps/modules/amazon-catalog-helpers.ts b/src/apps/catalog/common/catalog-helpers.ts similarity index 95% rename from src/apps/modules/amazon-catalog-helpers.ts rename to src/apps/catalog/common/catalog-helpers.ts index c9c6606..f7720c3 100644 --- a/src/apps/modules/amazon-catalog-helpers.ts +++ b/src/apps/catalog/common/catalog-helpers.ts @@ -1,25 +1,25 @@ -import type { AmazonGetItem } from '../../data/models/multis/AmazonGetItem.ts'; -import type { Brand } from '../../data/models/multis/Brand.ts'; -import type { ContentsDto } from '../../data/internals/ContentsDtoT.ts'; +import type { AmazonGetItem } from '../../../data/models/multis/AmazonGetItem.ts'; +import type { Brand } from '../../../data/models/multis/Brand.ts'; +import type { ContentsDto } from '../../../data/internals/ContentsDtoT.ts'; import type { GetItemsResponse, Item } from 'amazon-pa-api5-node-ts'; -import type { Listing } from '../../data/models/multis/Listing.ts'; -import type { Multilingual } from '../../data/internals/MultilingualT.ts'; -import type { NonMultilingual } from '../../data/internals/NonMultilingualT.ts'; -import type { Product } from '../../data/models/multis/Product.ts'; -import type { ProductCategory } from '../../data/models/multis/ProductCategory.ts'; -import type { Seller } from '../../data/models/multis/Seller.ts'; +import type { Listing } from '../../../data/models/multis/Listing.ts'; +import type { Localized } from '../../../data/internals/LocalizedT.ts'; +import type { NonLocalized } from '../../../data/internals/NonLocalizedT.ts'; +import type { Product } from '../../../data/models/multis/Product.ts'; +import type { ProductCategory } from '../../../data/models/multis/ProductCategory.ts'; +import type { Seller } from '../../../data/models/multis/Seller.ts'; import { arrayBuffer } from 'stream/consumers'; -import { client, getContentsByIds, TIMEOUT_IN_SECONDS } from '../../data/core/client.ts'; +import { client, getContentsByIds, TIMEOUT_IN_SECONDS } from '../../../data/core/client.ts'; import { createReadStream, type ReadStream } from 'fs'; -import { getItems } from '../../apiclient/amazon.ts'; -import { getBrandsUsingJsonQuery, getMarketplacesUsingJsonQuery, getProductCategoriesUsingJsonQuery, getProductsUsingJsonQuery, getSellersUsingJsonQuery } from '../../data/api-client.ts'; +import { getItems } from '../../../apiclient/amazon.ts'; +import { getBrandsUsingJsonQuery, getMarketplacesUsingJsonQuery, getProductCategoriesUsingJsonQuery, getProductsUsingJsonQuery, getSellersUsingJsonQuery } from '../../../data/api-client.ts'; import { memfs } from 'memfs'; -import { response } from '../../old-data/brands/first-aid-only-example-query-getitems.ts'; -import { SCHEMAS } from '../../data/models/schemas.ts'; +import { response } from '../../../old-data/brands/first-aid-only-example-query-getitems.ts'; +import { SCHEMAS } from '../../../data/models/schemas.ts'; import axios from 'axios'; import ollama from 'ollama'; import slugify from 'slugify'; -import type { Offer } from '../../data/models/multis/Offer.ts'; +import type { Offer } from '../../../data/models/multis/Offer.ts'; export function isValidASIN(asinOrNot: string) { return asinOrNot.match(/[A-Z0-9]{10}/g) ? true : false; @@ -346,9 +346,9 @@ export async function designAiProductSubCategoryEvalQuestionsSet2(response: GetI } export async function askAiProductSubCategoryEvalQuestionsSet2(response: GetItemsResponse, parentProductCategoryId: string, brandName: string, productName_en_US: string, features: string[], shouldCreateNewProductCategory: boolean): Promise<{ - categoryName: Multilingual, - description: Multilingual, - parentCategory: NonMultilingual, + categoryName: Localized, + description: Localized, + parentCategory: NonLocalized, }|string> { const aiSubCategoryEvalQuestion2 = await designAiProductSubCategoryEvalQuestionsSet2(response, parentProductCategoryId, brandName, productName_en_US, features, shouldCreateNewProductCategory); console.log(`>>>${aiSubCategoryEvalQuestion2}`); @@ -358,8 +358,8 @@ export async function askAiProductSubCategoryEvalQuestionsSet2(response: GetItem })).message.content; console.log(`llama3.1>${answer}`); const aiSubCategoryEvalQuestion2Answer = JSON.parse(answer) as { - categoryName: Multilingual, - description: Multilingual, + categoryName: Localized, + description: Localized, parentCategory: string[], }|string; if (typeof aiSubCategoryEvalQuestion2Answer === 'string') { @@ -406,7 +406,7 @@ export async function translateTags_from_en_US_to_fr_CA(tags_en_US: string[]) { return (JSON.parse(answer) as string[]).map(a => a.trim().toLowerCase()); } -export async function getAddNewProductSubCategoryDto(parentProductCategoryId: NonMultilingual, categoryName: Multilingual, description: Multilingual) { +export async function getAddNewProductSubCategoryDto(parentProductCategoryId: NonLocalized, categoryName: Localized, description: Localized) { let productCategoryDto = await client.contents.postContent(SCHEMAS.PRODUCT_CATEGORIES, { publish: false, body: { diff --git a/src/apps/catalog/common/console.ts b/src/apps/catalog/common/console.ts new file mode 100644 index 0000000..1a73425 --- /dev/null +++ b/src/apps/catalog/common/console.ts @@ -0,0 +1,9 @@ +/** + * Returns a log function which upon calling will prepend the name of the command to the output. You can specify + * any log function, or if you don't specify one it will choose `console.log`. + * @param commandName Name of command to prepend in the logs. + * @param logFn The log function. If you don't provide a log function, it will automatically choose `console.log`. + * @returns A log function which upon calling it will prepend the command name to each output line. + */ +export const logForCommand = (commandName: string, logFn: ((...args: any[]) => void) = console.log) => + (...args: any[]) => logFn.apply(null, [`[${commandName}]`, ...args]); \ No newline at end of file diff --git a/src/apps/catalog/products/products-ls.ts b/src/apps/catalog/products/products-ls.ts new file mode 100644 index 0000000..195df97 --- /dev/null +++ b/src/apps/catalog/products/products-ls.ts @@ -0,0 +1,31 @@ +import type { Command } from "commander"; +import { getProductsUsingJsonQuery } from "../../../data/api-client"; +import type { AmazonMarketplaceConnection } from "../../../data/models/components/AmazonMarketplaceConnection"; +import { logForCommand } from "../common/console"; + +export const COMMAND_NAME = 'ls'; + +const log = logForCommand(COMMAND_NAME); + +export const productsLsCommand = (productsCommands: Command) => { + const command = productsCommands.command(COMMAND_NAME).alias('list') + .description('List all the products') + .action(async (args: string[]) => { + let productsDto = await getProductsUsingJsonQuery(); + productsDto.items.forEach((productDto, index) => { + const logEach = logForCommand(`${COMMAND_NAME}:${index+1}`); + logEach(`ID: ${productDto.id}`) + logEach(`Product (en-US): ${productDto.data?.productName['en-US']}`); + logEach(`Product (es-US): ${productDto.data?.productName['es-US']}`); + logEach(`Product (fr-CA): ${productDto.data?.productName['fr-CA']}`); + let asin = productDto.data?.marketplaceConnections.iv.map((connection) => (connection.connection as AmazonMarketplaceConnection).asin).join(''); + let siteStripUrl = productDto.data?.marketplaceConnections.iv.map((connection) => (connection.connection as AmazonMarketplaceConnection).siteStripeUrl).join(''); + logEach(`ASIN: ${asin} ${!!(asin||'').match(/[A-Z0-9]{10}/g) ? 'Is a valid ASIN.' : 'Is not a valid ASIN.'}`) + logEach(`Amazon SiteStripe URL: ${siteStripUrl}`) + console.log(); + }); + log(`Returned ${productsDto.items.length} products.`) + }); + command.configureHelp(); + return command; +} \ No newline at end of file diff --git a/src/apps/catalog/products/products.ts b/src/apps/catalog/products/products.ts new file mode 100644 index 0000000..ca6d218 --- /dev/null +++ b/src/apps/catalog/products/products.ts @@ -0,0 +1,10 @@ +import type { Command } from "commander"; +import { productsLsCommand } from "./products-ls"; + +export const COMMAND_NAME = 'products'; + +export const productsCommand = (program: Command) => { + const productsCommand = program.command(COMMAND_NAME).description('Products commands'); + productsLsCommand(productsCommand); + return productsCommand; +} \ No newline at end of file diff --git a/src/apps/catalog/site/site-sync-slugs.ts b/src/apps/catalog/site/site-sync-slugs.ts new file mode 100644 index 0000000..52da25f --- /dev/null +++ b/src/apps/catalog/site/site-sync-slugs.ts @@ -0,0 +1,16 @@ +import type { Command } from "commander"; +import { getProductsUsingJsonQuery, performSyncLocalizedSlugs } from "../../../data/api-client"; +import type { AmazonMarketplaceConnection } from "../../../data/models/components/AmazonMarketplaceConnection"; +import { logForCommand } from "../common/console"; + +export const COMMAND_NAME = 'sync-slugs'; + +const log = logForCommand(COMMAND_NAME); + +export const siteSyncSlugsCommand = (siteCommands: Command) => + siteCommands.command(COMMAND_NAME) + .description('Sync URL slugs for each frontend endpoint.') + .action(async (asin: string, args: string[]) => { + await performSyncLocalizedSlugs(log); + }) + .configureHelp(); diff --git a/src/apps/catalog/site/site.ts b/src/apps/catalog/site/site.ts new file mode 100644 index 0000000..185852d --- /dev/null +++ b/src/apps/catalog/site/site.ts @@ -0,0 +1,10 @@ +import type { Command } from "commander"; +import { siteSyncSlugsCommand } from "./site-sync-slugs"; + +export const COMMAND_NAME = 'site'; + +export const siteCommands = (program: Command) => { + const siteCommands = program.command(COMMAND_NAME).description('Site commands'); + siteSyncSlugsCommand(siteCommands); + return siteCommands; +} \ No newline at end of file diff --git a/src/components/Banner.astro b/src/components/Banner.astro index 7f12436..b073dfa 100644 --- a/src/components/Banner.astro +++ b/src/components/Banner.astro @@ -1,13 +1,14 @@ --- -interface Banner { - editToken?: string, +import type { SquidexEditable } from './SharedProperties.js'; + +interface Banner extends SquidexEditable { homePageLink: string, siteName: string, } const { editToken, homePageLink, siteName } = Astro.props; --- -

{siteName}

+

{siteName}