dashersupply/src/apps/catalog/common/catalog-helpers.ts

494 lines
26 KiB
TypeScript

import type { AmazonGetItem } from '../../../data/models/multis/AmazonGetItem.ts';
import type { Brand } from '../../../data/models/multis/Brand.ts';
import type { ContentsDto } from '../../../data/internals/ContentsDtoT.ts';
import type { GetItemsResponse, Item } from 'amazon-pa-api5-node-ts';
import type { Listing } from '../../../data/models/multis/Listing.ts';
import type { Localized } from '../../../data/internals/LocalizedT.ts';
import type { NonLocalized } from '../../../data/internals/NonLocalizedT.ts';
import type { Product } from '../../../data/models/multis/Product.ts';
import type { ProductCategory } from '../../../data/models/multis/ProductCategory.ts';
import type { Seller } from '../../../data/models/multis/Seller.ts';
import { arrayBuffer } from 'stream/consumers';
import { client, getContentsByIds, TIMEOUT_IN_SECONDS } from '../../../data/core/client.ts';
import { createReadStream, type ReadStream } from 'fs';
import { getItems } from '../../../apiclient/amazon.ts';
import { getBrandsUsingJsonQuery, getMarketplacesUsingJsonQuery, getProductCategoriesUsingJsonQuery, getProductsUsingJsonQuery, getSellersUsingJsonQuery } from '../../../data/api-client.ts';
import { memfs } from 'memfs';
import { response } from '../../../old-data/brands/first-aid-only-example-query-getitems.ts';
import { SCHEMAS } from '../../../data/models/schemas.ts';
import axios from 'axios';
import ollama from 'ollama';
import slugify from 'slugify';
import type { Offer } from '../../../data/models/multis/Offer.ts';
// import { Blob } from 'buffer';
import { lookup as mimeLookup } from 'mime-types';
export function isValidASIN(asinOrNot: string) {
return asinOrNot.match(/[A-Z0-9]{10}/g) ? true : false;
}
export async function getAmazonGetItemsRequestSchemaDto() {
return await client.schemas.getSchema('amazon-pa-get-items-request');
}
export async function lookupAmazonASINs(asins: string[]) {
let amazonGetItemsRequestSchemaDto = await getAmazonGetItemsRequestSchemaDto();
let requestDate = new Date().toISOString();
//TODO: Implement API client
let apiResponse = await getItems(asins);
//TODO: disable when API client is ready: returns mock data for now
//particularly I want to log every Amazon lookup from now until forever in Squidex
// let apiResponse = response as GetItemsResponse;
let amazonGetItem: AmazonGetItem = {
apiResponse: { iv: apiResponse },
getItemsRequest: { iv: { schemaId: amazonGetItemsRequestSchemaDto.id, ItemIds: [
...((apiResponse.ItemsResult && apiResponse.ItemsResult.Items!.length > 0) ?
apiResponse.ItemsResult!.Items!.map((item) => { return { ItemId: item.ASIN! } }) :
asins.map((asin) => { return { ItemId: asin }}))
] } },
requestDate: { iv: requestDate }
}
let amazonGetItemDto = await client.contents.postContent(SCHEMAS.AMAZON_GET_ITEMS, { ...amazonGetItem }, { publish: true })
let amazonGetItemsDto = await getContentsByIds<AmazonGetItem>(SCHEMAS.AMAZON_GET_ITEMS, amazonGetItemDto.id);
return amazonGetItemsDto;
}
export async function getMarketplaceConnectionSchemaDto() {
return await client.schemas.getSchema('product-marketplace-connection');
}
export async function getAmazonMarketplaceConnectionSchemaDto() {
return await client.schemas.getSchema('product-marketplace-connection-amazon')
}
export async function getAmazonMarketplaceDto() {
return await getMarketplacesUsingJsonQuery(JSON.stringify({
filter: {
op: 'eq',
path: 'data.marketplaceName.en-US',
value: 'Amazon'
}
}));
}
export async function doesProductAlreadyExist(asin: string) {
return (await getProductsUsingJsonQuery(JSON.stringify({
filter: { not: { and: [{
op: 'empty',
path: 'data.marketplaceConnections.iv.connection.asin',
// value: 'Amazon'
}, {
op: 'eq',
path: 'data.marketplaceConnections.iv.marketplace',
eq: (await getAmazonMarketplaceDto()).items[0].id
}] } }
}))).items.length > 0;
}
export async function getBrandDtoByName(brandName: string) {
return (await getBrandsUsingJsonQuery(JSON.stringify({
filter: {
op: 'eq',
path: 'data.brandName.en-US',
value: brandName,
}
})));
}
export async function getAddNewBrandDtoByName(brandName: string) {
let brandDto = await client.contents.postContent(SCHEMAS.BRANDS, {
brandName: {
"en-US": brandName!,
"es-US": brandName!,
"fr-CA": brandName!
},
slug: {
"en-US": `en-US/${slugify(brandName!, { lower: true, trim: true })}`,
"es-US": `es-US/${slugify(brandName!, { lower: true, trim: true })}`,
"fr-CA": `fr-CA/${slugify(brandName!, { lower: true, trim: true })}`
},
});
let brandsDto = await client.contents.getContents(SCHEMAS.BRANDS, { unpublished: true, ids: brandDto.id }) as ContentsDto<Brand>;
return brandsDto;
}
export async function getSellerDtoByName(sellerName: string) {
return (await getSellersUsingJsonQuery(JSON.stringify({
filter: {
op: 'eq',
path: 'data.sellerName.en-US',
value: sellerName,
}
})));
}
export async function getAddNewSellerDtoByName(sellerName: string) {
let sellerDto = await client.contents.postContent(SCHEMAS.SELLERS, {
sellerName: {
"en-US": sellerName!,
"es-US": sellerName!,
"fr-CA": sellerName!
},
slug: {
"en-US": `en-US/${slugify(sellerName!, { lower: true, trim: true })}`,
"es-US": `es-US/${slugify(sellerName!, { lower: true, trim: true })}`,
"fr-CA": `fr-CA/${slugify(sellerName!, { lower: true, trim: true })}`
},
});
let sellersDto = await client.contents.getContents(SCHEMAS.SELLERS, { unpublished: true, ids: sellerDto.id }) as ContentsDto<Seller>;
return sellersDto;
}
export async function translateProductName_from_en_US_to_es_US(item: Item, brandName: string, productName_en_US: string) {
return (await ollama.chat({
model: 'llama3.1',
messages: [{
role: 'user',
content:
`Translate just the product name "${productName_en_US}" into Latin American Spanish, formal, no commentary, no alternatives, ignoring the rest of this text.` +
`The brand name "${brandName}" remains unchanged. Here is the full product description for reference only:\n` +
`${item.ItemInfo?.Features?.DisplayValues?.map((item) => `- ${item}`).join('\n')}\n`,
}]
})).message.content;
}
export async function translateProductName_from_en_US_to_fr_CA(item: Item, brandName: string, productName_en_US: string) {
return (await ollama.chat({
model: 'llama3.1',
messages: [{
role: 'user',
content:
`Translate just the product name "${productName_en_US}" into French Canadian, formal, no commentary, no alternatives, ignoring the rest of this text.` +
`The brand name "${brandName}" remains unchanged. Here is the full product description for reference only:\n` +
`${item.ItemInfo?.Features?.DisplayValues?.map((item) => `- ${item}`).join('\n')}\n`,
}]
})).message.content;
}
export async function generateAIProductDescription(productName_en_US: string, features: string[]) {
return (await ollama.chat({
model: 'llama3.1',
messages: [{
role: 'user',
content:
`I have a niche wish style website called Dasher Supply. It contains many products which are great picks for delivery drivers. `+
`I need you to write me a product pitch based on the information the following rules: 1. please don't bother with introductions and conclusions, `+
`those are implied because these are articles on the website; 2. please, whatever you do, do not invent any new information about the products--you should `+
`only ever produce outputs based on the inputs; 3. if you aren't sure whether to call us Dashers or delivery drivers, do use delivery drivers or drivers, etc., `+
`as I do not want to cause any issues with the actual DoorDash company; 4. please no commentary, just provide the review based on what you know, but try`+
`to sell it to a delivery driver. Here is the product name:`+ '`' + productName_en_US + '`. Here are the product features as generally marketed on Amazon: '+
`${features.map((feature) => `- ${feature}`).join('\n')}.`
}]
})).message.content;
}
export async function translateProductDescription_from_en_US_to_es_US(brandName: string, productDescription_en_US: string) {
return (await ollama.chat({
model: 'llama3.1',
messages: [{
role: 'user',
content:
`I have a product description in United States English that I need to translate to Latin American Spanish. If you aren't sure about something ` +
`please use your best judgement in selecting alternatives. The speech can be informal but we're mostly professionals selling products and we ` +
`want to include options for alternative languages. Please no commentary, just the translation. The brand is ${brandName}, don't translate it. Here is the description in English:\n` +
productDescription_en_US
}]
})).message.content;
}
export async function translateProductDescription_from_en_US_to_fr_CA(brandName: string, productDescription_en_US: string) {
return (await ollama.chat({
model: 'llama3.1',
messages: [{
role: 'user',
content:
`I have a product description in United States English that I need to translate to French Canadian. If you aren't sure about something ` +
`please use your best judgement in selecting alternatives. The speech can be informal but we're mostly professionals selling products and we ` +
`want to include options for alternative languages. Please no commentary such as "Here is the translation to French Canadian:", just the text translation. `+
`I need to pipe the output of this command. The brand is ${brandName}, don't translate it. Here is the description in English:\n` +
productDescription_en_US
}]
})).message.content;
}
export async function getAllTopLevelProductCategories() {
return (await getProductCategoriesUsingJsonQuery()).items
.filter((category) =>
!!category.data?.parentCategory.iv || !category.data?.parentCategory.iv.length)
.map((category) => {
return { id: category.id, categoryName: category.data?.categoryName, description: category.data?.description, parentCategory: category.data?.parentCategory };
});
}
export async function getAllProductSubCategories(parentProductCategoryId: string) {
return (await getProductCategoriesUsingJsonQuery(JSON.stringify({ filter: { path: 'data.parentCategory.iv', op: 'eq', value: parentProductCategoryId } }))).items.map((category) => { return { id: category.id, categoryName: category.data?.categoryName, description: category.data?.description, parentCategory: category.data?.parentCategory||undefined }; });
}
export async function designAiTopLevelProductCategoryEvalQuestions(brandName: string, productName_en_US: string) {
const topLevelProductCategories = await getAllTopLevelProductCategories();
return [
// prompt 1
`My website is called Dasher Supply. It contains products and product categories. I need your help in categorizing a product.`,
// prompt 2
`I have ${topLevelProductCategories.length} top level product categories. The top level product categories I have already are (in JSON format):\n\n` +
JSON.stringify(topLevelProductCategories),
// prompt 3
`Which of these top level product categories should I pick to organize this product? ` +
`The brand name for the product is ${brandName}. The name of the product (in en-US) is ${productName_en_US}. ` +
`The product features (in US English) are:\n` +
`${response.ItemsResult?.Items![0].ItemInfo?.Features?.DisplayValues?.map((item) => `- ${item}`).join('\n')}\n`,
// prompt 4
`I need the answer in a certain format. In this case I must have only the category UUID as a properly escaped valid JavaScript string in double-quotes, no commentary.`,
];
}
export async function askAiTopLevelProductCategoryEvalQuestions(brandName: string, productName_en_US: string) {
const aiTopLevelCategoryEvalQuestions: string[] = await designAiTopLevelProductCategoryEvalQuestions(brandName, productName_en_US);
return JSON.parse((await ollama.chat({
model: 'llama3.1',
messages: aiTopLevelCategoryEvalQuestions.map(content => { return { role: 'user', content } }),
})).message.content).trim().toLowerCase() as string;
}
export async function designAiProductSubCategoryEvalQuestionsSet1(parentProductCategoryId: string, brandName: string, productName_en_US: string, features: string[]) {
const topLevelProductCategories = await getAllTopLevelProductCategories();
const productSubCategories = await getAllProductSubCategories(parentProductCategoryId);
const aiTopLevelCategoryEvalQuestions: string[] = await designAiTopLevelProductCategoryEvalQuestions(brandName, productName_en_US);
return [
// prompt 1
aiTopLevelCategoryEvalQuestions[0],
// prompt 2
aiTopLevelCategoryEvalQuestions[1],
// prompt 3
`The brand name for the product is ${brandName}. The name of the product (in en-US) is ${productName_en_US}. The product features (in US English) are:\n` +
`${features.map((item) => `- ${item}`).join('\n')}\n` +
`When asked previously about which top level product category, you picked ${parentProductCategoryId} as the closest match.`,
// prompt 4
`The second level product categories I have already are (in JSON format):\n\n` +
JSON.stringify(productSubCategories) +
` Ignoring the top level product categories, should I pick an existing second level category from the list (if any), or make a new one?`,
// prompt 5
`I need the answer in a certain format. In this case I must have only the answer as a valid JavaScript boolean to read in with JSON.parse. If you suggest making a new second level category, return true, no commentary. If you suggest using an existing second level category, return false, no commentary.`
];
}
export async function askAiProductSubCategoryEvalQuestionsSet1(parentProductCategoryId: string, brandName: string, productName_en_US: string, features: string[]) {
const aiSubCategoryEvalQuestion1 = await designAiProductSubCategoryEvalQuestionsSet1(parentProductCategoryId, brandName, productName_en_US, features);
return JSON.parse((await ollama.chat({
model: 'llama3.1',
messages: aiSubCategoryEvalQuestion1.map(content => { return { role: 'user', content } }),
})).message.content) as boolean;
}
export async function designAiProductSubCategoryEvalQuestionsSet2(response: GetItemsResponse, parentProductCategoryId: string, brandName: string, productName_en_US: string, features: string[], shouldCreateNewProductCategory: boolean) {
const topLevelProductCategories = await getAllTopLevelProductCategories();
const productSubCategories = await getAllProductSubCategories(parentProductCategoryId);
const aiTopLevelCategoryEvalQuestions: string[] = await designAiTopLevelProductCategoryEvalQuestions(brandName, productName_en_US);
return [
// prompt 1
aiTopLevelCategoryEvalQuestions[0],
// prompt 2
aiTopLevelCategoryEvalQuestions[1],
// prompt 3
`The brand name for the product is ${brandName}. The name of the product (in en-US) is ${productName_en_US}. The product features (in US English) are:\n` +
`${response.ItemsResult?.Items![0].ItemInfo?.Features?.DisplayValues?.map((item) => `- ${item}`).join('\n')}\n` +
`When asked previously about which top level product category, you picked ${parentProductCategoryId} as the closest match.`,
// prompt 4
`The second level product categories I have already are (in JSON format):\n\n` +
JSON.stringify(productSubCategories) +
` When asked whether I should pick an existing second level category from the list (if any) or make a new one you previously responded ${shouldCreateNewProductCategory?" that I should make a new one.":" that I should choose an existing one."}`,
// prompt 5
shouldCreateNewProductCategory
?
`What would be a good subcategory to use for this product based on what you know about the product, the existing categories, and the web site?`
:
`Which existing subcategory do you think I should use?`
,
// prompt 6
shouldCreateNewProductCategory
?
`In TypeScript, the interface looks like this:\n\n` +
`internals/NonMultilingualT.ts:` + "```\n" +
`export interface NonMultilingual<T> {\n` +
` [key: string]: T,\n` +
` iv: T,\n` +
`};\n` +
"```\n" +
`internals/MultilingualT.ts:` + "```" +
`export interface Multilingual<T> {\n` +
` [key: string]: T,\n` +
` 'en-US': T,\n` +
` 'es-US': T,\n` +
` 'fr-CA': T,\n` +
`};\n` +
"```\n" +
`models/multis/ProductCategory.ts:` + "```\n" +
`import type { Multilingual } from "../../internals/MultilingualT";\n` +
`import type { NonMultilingual } from "../../internals/NonMultilingualT";\n` +
`export interface ProductCategory {\n` +
` categoryName: Multilingual<string>, //multilingual name of category` +
` description: Multilingual<string>, //multilingual description of category` +
` parentCategory: NonMultilingual<string[]>, //parent category (exactly 1), though due to Squidex / MongoDB this is always an array of UUIDs with the UUID belonging to the UUID of the parent category or an empty array for top-level categories` +
`}` + "```\n\n" +
`I need the answer in a certain format. In this case I must have only the JSON for the new Category containing only the fields categoryName and description in all three languages, no commentary, no code comments, and with the UUID in the string array for parentCategory field, no commentary, no code comments, no id field, not in a markdown codeblock.`
:
`I need the answer in a certain format. In this case I must have only the category UUID as a properly escaped valid JavaScript string, no commentary.`,
];
}
export async function askAiProductSubCategoryEvalQuestionsSet2(response: GetItemsResponse, parentProductCategoryId: string, brandName: string, productName_en_US: string, features: string[], shouldCreateNewProductCategory: boolean): Promise<{
categoryName: Localized<string>,
description: Localized<string>,
parentCategory: NonLocalized<string[]>,
}|string> {
const aiSubCategoryEvalQuestion2 = await designAiProductSubCategoryEvalQuestionsSet2(response, parentProductCategoryId, brandName, productName_en_US, features, shouldCreateNewProductCategory);
console.log(`>>>${aiSubCategoryEvalQuestion2}`);
const answer = (await ollama.chat({
model: 'llama3.1',
messages: aiSubCategoryEvalQuestion2.map(content => { return { role: 'user', content } }),
})).message.content;
console.log(`llama3.1>${answer}`);
const aiSubCategoryEvalQuestion2Answer = JSON.parse(answer) as {
categoryName: Localized<string>,
description: Localized<string>,
parentCategory: string[],
}|string;
if (typeof aiSubCategoryEvalQuestion2Answer === 'string') {
return aiSubCategoryEvalQuestion2Answer as string;
} else {
return {
categoryName: aiSubCategoryEvalQuestion2Answer.categoryName,
description: aiSubCategoryEvalQuestion2Answer.description,
parentCategory: { iv: aiSubCategoryEvalQuestion2Answer.parentCategory },
};
}
}
export async function generateAITagsForProduct(productName_en_US: string, features: string[]) {
const questions: string[] = [
`Provided the description for the product, please provide a valid JavaScript array of strings which represent tags on a website for the product, no commentary, no code comments, not in a markdown codeblock. The product name is ${productName_en_US}. The product features are:\n ${features.map((feature) => `- ${feature}`).join('\n')}`
];
let answer = (await ollama.chat({
model: 'llama3.1',
messages: questions.map(content => { return { role: 'user', content } }),
})).message.content;
return (JSON.parse(answer) as string[]).map(a => a.trim().toLowerCase());
}
export async function translateTags_from_en_US_to_es_US(tags_en_US: string[]) {
const questions: string[] = [
`Provided these website tags in United States English in the following JavaScript array of strings, please provide a valid JavaScript array of strings which represent the tags translated to Latin American Spanish, no commentary, no code comments, not in a markdown codeblock. The tags are:\n ${JSON.stringify(tags_en_US)}`
];
let answer = (await ollama.chat({
model: 'llama3.1',
messages: questions.map(content => { return { role: 'user', content } }),
})).message.content;
return (JSON.parse(answer) as string[]).map(a => a.trim().toLowerCase());
}
export async function translateTags_from_en_US_to_fr_CA(tags_en_US: string[]) {
const questions: string[] = [
`Provided these website tags in United States English in the following JavaScript array of strings, please provide a valid JavaScript array of strings which represent the tags translated to French Canadian, no commentary, no code comments, not in a markdown codeblock. The tags are:\n ${JSON.stringify(tags_en_US)}`
];
let answer = (await ollama.chat({
model: 'llama3.1',
messages: questions.map(content => { return { role: 'user', content } }),
})).message.content;
return (JSON.parse(answer) as string[]).map(a => a.trim().toLowerCase());
}
export async function getAddNewProductSubCategoryDto(parentProductCategoryId: NonLocalized<string[]>, categoryName: Localized<string>, description: Localized<string>) {
let productCategoryDto = await client.contents.postContent(SCHEMAS.PRODUCT_CATEGORIES, {
categoryName,
description,
parentCategory: parentProductCategoryId,
}, {
publish: false,
});
let productCategoriesDto = await client.contents.getContents(SCHEMAS.PRODUCT_CATEGORIES, { unpublished: true, ids: productCategoryDto.id }) as ContentsDto<ProductCategory>;
return productCategoriesDto;
}
export async function translateAmazonDescription_from_en_US_to_es_US(brandName: string, features: string[]) {
return (await ollama.chat({
model: 'llama3.1',
messages: [{
role: 'user',
content:
`Translate the Amazon description into Latin American Spanish, formal, no commentary, no alternatives. ` +
`The brand name ${brandName} remains unchanged. Here is the description:\n` +
`${features.map((feature) => `- ${feature}`).join('\n')}\n`,
}]
})).message.content;
}
export async function translateAmazonDescription_from_en_US_to_fr_CA(brandName: string, features: string[]) {
return (await ollama.chat({
model: 'llama3.1',
messages: [{
role: 'user',
content:
`Translate the Amazon description into French Canadian, formal, no commentary, no alternatives. ` +
`The brand name ${brandName} remains unchanged. Here is the description:\n` +
`${features.map((feature) => `- ${feature}`).join('\n')}\n`,
}]
})).message.content;
}
export async function getAddNewProductDtoByProduct(product: Product) {
let productDto = await client.contents.postContent(SCHEMAS.PRODUCTS, { ...product }, { publish: false });
let productsDto = await client.contents.getContents(SCHEMAS.PRODUCTS, { unpublished: true, ids: productDto.id }) as ContentsDto<Product>;
return productsDto;
}
export async function getAddNewProductListingDtoByProduct(listing: Listing) {
let listingDto = await client.contents.postContent(SCHEMAS.LISTINGS, { ...listing }, {
publish: true,
});
let listingsDto = await client.contents.getContents(SCHEMAS.LISTINGS, { unpublished: true, ids: listingDto.id }) as ContentsDto<Listing>;
return listingsDto;
}
export function trimPeriods(str: string) {
return str.replace(/^\s+|\s+$/g, '').replace(/\.$/g, '');
}
export function removeQuotes(str: string) {
return str.replace(/['"]+/g, '');
}
export async function upsertAssetFolder(folderName: string, parentFolderId?: string|undefined) {
const assetFolders = await client.assets.getAssetFolders({ scope: 'Items', parentId: parentFolderId });
let assetFolder;
let assetFolderLookup = assetFolders.items.filter(folder => folder.folderName === folderName);
if (assetFolderLookup.length === 0) {
assetFolder = await client.assets.postAssetFolder({ folderName: folderName, parentId: parentFolderId });
}
else {
assetFolder = assetFolderLookup[0];
}
return assetFolder;
}
export async function getAllAssetsInFolder(assetFolderId: string) {
let assetsDto = await client.assets.getAssets({parentId: assetFolderId});
return assetsDto.items||[];
}
export async function uploadDownloadedImageToSquidexAsAsset(downloadUrl: string, assetFolderId: string) {
let url = new URL(downloadUrl);
let filename = url.pathname.substring(url.pathname.lastIndexOf('/')+1);
let res = await fetch(downloadUrl);
let ab = await res.arrayBuffer();
let file = new File([ab], filename, { type: mimeLookup(filename) as string });
let assetDto = await client.assets.postAsset({ file, parentId: assetFolderId });
assetDto = await client.assets.putAsset(assetDto.id, { metadata: { ...assetDto.metadata, 'amazon-url': downloadUrl }, tags: ['amazon', 'product'] });
return assetDto;
}
export async function getAddNewOfferDto(offer: Offer) {
let offerDto = await client.contents.postContent(SCHEMAS.OFFERS, { ...offer }, { publish: true });
let offersDto = await client.contents.getContents(SCHEMAS.OFFERS, { unpublished: true, ids: offerDto.id }) as ContentsDto<ProductCategory>;
return offersDto;
}