feat: get variation categories automatically

src/lib/server/crawler/clevertronik/data/battery.ts

@@ -0,0 +1,13 @@
+import { Effect } from 'effect'
+import type { Page } from 'puppeteer'
+import { getSelected } from '.'
+import { getBatteryVariations } from '../variations/battery'
+
+export const getBattery = (page: Page) =>
+  Effect.gen(function* () {
+    const batteryVariations = yield* getBatteryVariations(page)
+
+    const selectedCapacity = yield* getSelected(batteryVariations ?? [])
+
+    return selectedCapacity.label
+  })

src/lib/server/crawler/clevertronik/data/index.ts

@@ -9,6 +9,7 @@ import { getColor } from './color'
 import { getSim } from './sim'
 import { getStockLevel } from './stockLevel'
 import { getDevices } from './devices'
+import { getBattery } from './battery'
 
 export class ExtractSelectedVariationError extends Data.TaggedError(
   'ExtractSelectedVariationError',
@@ -49,6 +50,7 @@ export type PageData = {
   color: string
   sim: string
   stockLevel: string
+  battery: string
   devices: DeviceData[]
 }
 
@@ -76,8 +78,11 @@ export const getPageData = (page: Page) =>
     const sim = yield* getSim(page)
 
     const stockLevel = yield* getStockLevel(page)
+
     const devices = yield* getDevices(page)
 
+    const battery = yield* getBattery(page)
+
     const pageData: PageData = {
       price,
       productName,
@@ -87,6 +92,7 @@ export const getPageData = (page: Page) =>
       sim,
       stockLevel,
       devices,
+      battery,
     }
 
     return pageData

src/lib/server/crawler/clevertronik/index.ts

@@ -1,14 +1,15 @@
-import { Data, Effect, pipe } from 'effect'
+import { Data, Effect } from 'effect'
 import { openPage } from '../openPage'
 import * as Crawler from '../index'
 import { withOperation } from '$lib/server/utils'
 import { getProductName } from './data/productName'
-import { getAllVariations, VariationEnum, type Variation } from './variations'
+import {
-import { getPageData, type PageData } from './data'
+  getAllVariations,
-import type { Page } from 'puppeteer'
+  getVariationCategory,
+  VariationEnum,
+} from './variations'
 import {
   getAllVariationCategoryData,
-  getAllVariationData,
   type VariationCategory,
 } from './variationData'
 
@@ -30,37 +31,9 @@ export const crawlClevertronik = Effect.gen(function* () {
 
   yield* Effect.logInfo('opened clevertronik page')
 
-  let productName = yield* getProductName(page)
+  const variationCategory = yield* getVariationCategory(page)
 
-  const variations = yield* getAllVariations(page)
+  yield* Effect.logInfo(`Got tree`, { variationCategory })
-
-  // const variationCategory: VariationCategory = {
-  //   name: 'Capacity',
-  //   variations: variations.capacities,
-  //   subVariationCategory: {
-  //     name: 'Color',
-  //     variations: variations.colors,
-  //     subVariationCategory: {
-  //       name: 'Sim',
-  //       variations: variations.sims,
-  //       subVariationCategory: {
-  //         name: 'Condition',
-  //         variations: variations.conditions,
-  //         subVariationCategory: {
-  //           name: 'Battery',
-  //           variations: variations.battery,
-  //         },
-  //       },
-  //     },
-  //   },
-  // }
-
-  const variationCategory: VariationCategory = {
-    name: VariationEnum.CAPACITY,
-    subVariationCategory: {
-      name: VariationEnum.COLOR,
-    },
-  }
 
   const variationData = yield* getAllVariationCategoryData(
     page,
@@ -70,9 +43,11 @@ export const crawlClevertronik = Effect.gen(function* () {
   yield* Effect.promise(() => page.browser().close())
 
   return {
-    productName,
+    variationCategory,
-    ...variations,
     variationData,
+    // productName,
+    // ...variations,
+    // variationData,
   }
 }).pipe(
   Effect.provide(Crawler.CrawlerLayer()),

src/lib/server/crawler/clevertronik/variationData/index.ts

@@ -34,7 +34,7 @@ export const getAllVariationCategoryData = (
     const variations =
       yield* getVariationHandler[variationCategory.name](variationPage)
 
-    for (const variation of variations.filter((v) => v.active)) {
+    for (const variation of variations?.filter((v) => v.active) ?? []) {
       if (!variation.selected) {
         yield* gotoVariation(variationPage, variation)
       }

src/lib/server/crawler/clevertronik/variations/battery.ts

@@ -6,5 +6,7 @@ const BATTERY_CONTAINER_SELECTOR = '.new_battery_options'
 
 export const getBatteryVariations = (page: Page) =>
   Effect.gen(function* () {
-    return yield* getVariations(page, BATTERY_CONTAINER_SELECTOR, 'DIV')
+    return yield* getVariations(page, BATTERY_CONTAINER_SELECTOR, 'DIV').pipe(
+      Effect.catchTag('VariationParseError', () => Effect.succeed(undefined)),
+    )
   })

src/lib/server/crawler/clevertronik/variations/index.ts

@@ -6,6 +6,7 @@ import { getColorVariations } from './color'
 import { getSimVariations } from './sim'
 import { getConditionVariations } from './condition'
 import { getBatteryVariations } from './battery'
+import type { VariationCategory } from '../variationData'
 
 export type Variation = {
   label: string
@@ -27,12 +28,19 @@ export class VariationParseError extends Data.TaggedError(
   message?: string
 }> {}
 
+export class VariationCategoryError extends Data.TaggedError(
+  'VariationCategoryError',
+)<{
+  cause?: unknown
+  message?: string
+}> {}
+
 export enum VariationEnum {
   CAPACITY = 'Capacity',
   COLOR = 'Color',
   SIM = 'Sim',
   CONDITION = 'CONDITION',
-  BATTERY = 'BATTER',
+  BATTERY = 'BATTERY',
 }
 
 export const getVariationHandler = {
@@ -116,3 +124,96 @@ export const getAllVariations = (page: Page) =>
       battery,
     }
   })
+
+const attributeVariationMapping: Record<string, VariationEnum> = {
+  prop_19: VariationEnum.CAPACITY,
+  prop_17: VariationEnum.COLOR,
+  prop_39: VariationEnum.SIM,
+  condition: VariationEnum.CONDITION,
+}
+
+export const getVariationCategory = (page: Page) =>
+  Effect.gen(function* () {
+    const attributes: string[] = yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate(() => {
+          const divs = document.querySelectorAll('[data-scrollel]')
+
+          const attributes: string[] = []
+
+          divs.forEach((div) => {
+            const val = div.getAttribute('data-scrollel')
+            if (!val) return
+            attributes.push(val)
+          })
+
+          return attributes
+        }),
+      catch: (cause) =>
+        new VariationParseError({
+          message: 'Could not get variation attributes',
+          cause,
+        }),
+    })
+
+    const hasBatteryVariation = yield* Effect.tryPromise({
+      try: () =>
+        page.evaluate(() => {
+          const batteryDiv: HTMLDivElement | null = document.querySelector(
+            '.new_battery_options',
+          )
+
+          return Boolean(batteryDiv)
+        }),
+      catch: (cause) =>
+        new VariationParseError({
+          message: 'Error getting possible battery variation',
+          cause,
+        }),
+    })
+
+    let variationCategory: VariationCategory | undefined
+    let currentCategory: VariationCategory | undefined
+
+    for (const attr of attributes) {
+      const variationKey = attributeVariationMapping[attr]
+      if (!variationKey) continue
+
+      if (!variationCategory) {
+        variationCategory = {
+          name: variationKey,
+        }
+
+        currentCategory = variationCategory
+        continue
+      }
+
+      if (!currentCategory) {
+        continue
+      }
+
+      const newCategory: VariationCategory = {
+        name: variationKey,
+      }
+
+      currentCategory.subVariationCategory = newCategory
+
+      currentCategory = newCategory
+    }
+
+    if (currentCategory && hasBatteryVariation) {
+      currentCategory.subVariationCategory = {
+        name: VariationEnum.BATTERY,
+      }
+    }
+
+    if (!variationCategory) {
+      return yield* Effect.fail(
+        new VariationCategoryError({
+          message: 'Unable to get variation category tree',
+        }),
+      )
+    }
+
+    return variationCategory
+  })