Refactor to external scraper and update dependencies (#113)

pages/[[...slug]].tsx

@@ -1,125 +1,191 @@
-import { useEffect, useReducer, useCallback, FC, ChangeEvent } from "react";
-import { GetStaticPaths, GetStaticProps, InferGetStaticPropsType } from "next";
-import Router from "next/router";
+import { useCallback, useEffect, useReducer } from "react";
+import { GetStaticPaths, GetStaticProps, NextPage } from "next";
+import { useRouter } from "next/router";
 import dynamic from "next/dynamic";
-import { Stack, VStack, HStack, IconButton } from "@chakra-ui/react";
+import {
+    getTranslationInfo,
+    getTranslationText,
+    getAudio,
+    languageList,
+    LanguageType,
+    replaceExceptedCode,
+    isValidCode,
+    TranslationInfo,
+    LangCode
+} from "lingva-scraper";
+import { HStack, IconButton, Stack, VStack } from "@chakra-ui/react";
 import { FaExchangeAlt } from "react-icons/fa";
 import { HiTranslate } from "react-icons/hi";
 import { useHotkeys } from "react-hotkeys-hook";
 import { CustomHead, LangSelect, TranslationArea } from "@components";
 import { useToastOnLoad } from "@hooks";
-import { googleScrape, extractSlug, textToSpeechScrape } from "@utils/translate";
-import { retrieveFromType, replaceBoth, isValid } from "@utils/language";
-import langReducer, { Actions, initialState } from "@utils/reducer";
+import { extractSlug } from "@utils/slug";
+import langReducer, { Actions, initialState, State } from "@utils/reducer";
 import { localGetItem, localSetItem } from "@utils/storage";
 
 const AutoTranslateButton = dynamic(() => import("@components/AutoTranslateButton"), { ssr: false });
 
-const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, translationRes, audio, errorMsg, initial }) => {
-    const [{ source, target, query, delayedQuery, translation, isLoading }, dispatch] = useReducer(langReducer, initialState);
+export enum ResponseType {
+    SUCCESS,
+    ERROR,
+    HOME
+}
 
-    const handleChange = (e: ChangeEvent<HTMLTextAreaElement | HTMLSelectElement>) => {
+type Props = {
+    type: ResponseType.SUCCESS,
+    translation: string,
+    info: TranslationInfo | null,
+    audio: {
+        query: number[] | null,
+        translation: number[] | null
+    },
+    initial: {
+        source: LangCode<"source">,
+        target: LangCode<"target">,
+        query: string
+    }
+} | {
+    type: ResponseType.ERROR,
+    errorMsg: string,
+    initial: {
+        source: LangCode<"source">,
+        target: LangCode<"target">,
+        query: string
+    }
+} | {
+    type: ResponseType.HOME
+};
+
+const Page: NextPage<Props> = (props) => {
+    const [
+        { source, target, query, delayedQuery, translation, isLoading, pronunciation, audio },
+        dispatch
+    ] = useReducer(langReducer, initialState);
+
+    const router = useRouter();
+
+    const setField = useCallback(<T extends keyof State,>(key: T, value: State[T]) => (
+        dispatch({ type: Actions.SET_FIELD, payload: { key, value }})
+    ), []);
+
+    const setAllFields = useCallback((state: State) => (
+        dispatch({ type: Actions.SET_ALL, payload: { state }})
+    ), []);
+
+    const setLanguage = useCallback((type: typeof LanguageType[keyof typeof LanguageType], code: string) => (
         dispatch({
-            type: Actions.SET_FIELD,
-            payload: {
-                key: e.target.id,
-                value: e.target.value
-            }
-        });
-    };
+            type: type === LanguageType.SOURCE
+                ? Actions.SET_SOURCE
+                : Actions.SET_TARGET,
+            payload: { code }
+        })
+    ), []);
+
+    const switchLanguages = useCallback((detectedSource?: LangCode<"source">) => (
+        dispatch({ type: Actions.SWITCH_LANGS, payload: { detectedSource } })
+    ), []);
 
     const changeRoute = useCallback((customQuery: string) => {
-        if (isLoading)
+        if (isLoading || router.isFallback)
             return;
         if (!customQuery || customQuery === initialState.query)
             return;
-        if (!home && !initial)
-            return;
-        if (!home && customQuery === initial.query && source === initial.source && target === initial.target)
+        if (props.type === ResponseType.SUCCESS && customQuery === props.initial.query
+         && source === props.initial.source && target === props.initial.target)
             return;
 
-        localSetItem("source", source);
-        localSetItem("target", target);
+        localSetItem(LanguageType.SOURCE, source);
+        localSetItem(LanguageType.TARGET, target);
 
-        dispatch({ type: Actions.SET_FIELD, payload: { key: "isLoading", value: true }});
-        Router.push(`/${source}/${target}/${encodeURIComponent(customQuery)}`);
-    }, [isLoading, source, target, home, initial]);
+        setField("isLoading", true);
+        router.push(`/${source}/${target}/${encodeURIComponent(customQuery)}`);
+    }, [isLoading, source, target, props, router, setField]);
 
     useEffect(() => {
-        if (home) {
-            const localSource = localGetItem("source");
-            const localTarget = localGetItem("target");
-            return dispatch({
-                type: Actions.SET_ALL,
-                payload: {
-                    state: {
-                        ...initialState,
-                        source: isValid(localSource) ? localSource : initialState.source,
-                        target: isValid(localTarget) ? localTarget : initialState.target,
-                        isLoading: false
-                    }
-                }
+        if (router.isFallback)
+            return;
+
+        if (props.type === ResponseType.HOME) {
+            const localSource = localGetItem(LanguageType.SOURCE);
+            const localTarget = localGetItem(LanguageType.TARGET);
+
+            return setAllFields({
+                ...initialState,
+                source: isValidCode(localSource, LanguageType.SOURCE)
+                    ? localSource
+                    : initialState.source,
+                target: isValidCode(localTarget, LanguageType.TARGET)
+                    ? localTarget
+                    : initialState.target,
+                isLoading: false
             });
         }
 
-        if (!initial)
-            return;
+        if (props.type === ResponseType.ERROR)
+            return setAllFields({
+                ...initialState,
+                ...props.initial,
+                delayedQuery: props.initial.query,
+                isLoading: false
+            });
 
-        dispatch({
-            type: Actions.SET_ALL,
-            payload: {
-                state: {
-                    ...initial,
-                    delayedQuery: initial.query,
-                    translation: translationRes,
-                    isLoading: false
-                }
+        setAllFields({
+            ...props.initial,
+            delayedQuery: props.initial.query,
+            translation: props.translation,
+            isLoading: false,
+            pronunciation: props.info?.pronunciation ?? {},
+            audio: {
+                query: props.audio.query ?? undefined,
+                translation: props.audio.translation ?? undefined
             }
         });
-    }, [initial, translationRes, home]);
+    }, [props, router, setAllFields]);
 
     useEffect(() => {
-        const timeout = setTimeout(() =>
-            dispatch({ type: Actions.SET_FIELD, payload: { key: "delayedQuery", value: query }}
-        ), 1000);
-        return () => clearTimeout(timeout);
-    }, [query]);
+        const timeoutId = setTimeout(() => setField("delayedQuery", query), 1000);
+        return () => clearTimeout(timeoutId);
+    }, [query, setField]);
 
     useEffect(() => {
         const handler = (url: string) => {
-            url === Router.asPath || dispatch({ type: Actions.SET_FIELD, payload: { key: "isLoading", value: true }});
+            url === router.asPath || setField("isLoading", true);
 
             if (url !== "/")
                 return;
-            dispatch({ type: Actions.SET_FIELD, payload: { key: "source", value: initialState.source }});
-            localSetItem("source", initialState.source);
-            dispatch({ type: Actions.SET_FIELD, payload: { key: "target", value: initialState.target }});
-            localSetItem("target", initialState.target);
+            setLanguage(LanguageType.SOURCE, initialState.source);
+            localSetItem(LanguageType.SOURCE, initialState.source);
+            setLanguage(LanguageType.TARGET, initialState.target);
+            localSetItem(LanguageType.TARGET, initialState.target);
         };
-        Router.events.on("beforeHistoryChange", handler);
-        return () => Router.events.off("beforeHistoryChange", handler);
-    }, []);
-
-    const sourceLangs = retrieveFromType("source");
-    const targetLangs = retrieveFromType("target");
-    const { source: transLang, target: queryLang } = replaceBoth("exception", { source: target, target: source });
+        router.events.on("beforeHistoryChange", handler);
+        return () => router.events.off("beforeHistoryChange", handler);
+    }, [router, setLanguage, setField]);
 
     useToastOnLoad({
-        title: "Unexpected error",
-        description: errorMsg,
         status: "error",
-        updateDeps: initial
+        title: "Unexpected error",
+        description: props.type === ResponseType.ERROR ? props.errorMsg : undefined,
+        updateDeps: props.type === ResponseType.ERROR ? props.initial : undefined
     });
 
-    const canSwitch = source !== "auto" && !isLoading;
+    const detectedSource = props.type === ResponseType.SUCCESS ? props.info?.detectedSource : undefined;
+
+    const canSwitch = !isLoading && (source !== "auto" || !!detectedSource);
 
     useHotkeys("ctrl+shift+s, command+shift+s, ctrl+shift+f, command+shift+f", () => (
-        canSwitch && dispatch({ type: Actions.SWITCH_LANGS })
-    ), [canSwitch]);
+        canSwitch && switchLanguages(detectedSource)
+    ), [canSwitch, detectedSource, switchLanguages]);
+
+    // parse existing code with opposite exceptions in order to flatten to the standards
+    const queryLang = source === "auto" && !!detectedSource
+        ? detectedSource
+        : replaceExceptedCode(LanguageType.TARGET, source);
+    const transLang = replaceExceptedCode(LanguageType.SOURCE, target);
 
     return (
         <>
-            <CustomHead home={home} />
+            <CustomHead home={props.type === ResponseType.HOME} />
 
             <VStack px={[8, null, 24, 40]} w="full">
                 <HStack px={[1, null, 3, 4]} w="full">
@@ -127,23 +193,24 @@ const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, transl
                         id="source"
                         aria-label="Source language"
                         value={source}
-                        onChange={handleChange}
-                        langs={sourceLangs}
+                        detectedSource={detectedSource}
+                        onChange={e => setLanguage(LanguageType.SOURCE, e.target.value)}
+                        langs={languageList.source}
                     />
                     <IconButton
                         aria-label="Switch languages"
                         icon={<FaExchangeAlt />}
                         colorScheme="lingva"
                         variant="ghost"
-                        onClick={() => dispatch({ type: Actions.SWITCH_LANGS })}
+                        onClick={() => switchLanguages(detectedSource)}
                         isDisabled={!canSwitch}
                     />
                     <LangSelect
                         id="target"
                         aria-label="Target language"
                         value={target}
-                        onChange={handleChange}
-                        langs={targetLangs}
+                        onChange={e => setLanguage(LanguageType.TARGET, e.target.value)}
+                        langs={languageList.target}
                     />
                 </HStack>
                 <Stack direction={["column", null, "row"]} w="full">
@@ -152,10 +219,11 @@ const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, transl
                         aria-label="Translation query"
                         placeholder="Text"
                         value={query}
-                        onChange={e => isLoading || handleChange(e)}
-                        onSubmit={useCallback(() => changeRoute(query), [query, changeRoute])}
+                        onChange={e => isLoading || setField("query", e.target.value)}
+                        onSubmit={() => changeRoute(query)}
                         lang={queryLang}
-                        audio={audio?.source}
+                        audio={audio.query}
+                        pronunciation={pronunciation.query}
                     />
                     <Stack direction={["row", null, "column"]} justify="center" spacing={3} px={[2, null, "initial"]}>
                         <IconButton
@@ -168,8 +236,9 @@ const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, transl
                             w={["full", null, "auto"]}
                         />
                         <AutoTranslateButton
-                            onAuto={useCallback(() => changeRoute(delayedQuery), [delayedQuery, changeRoute])}
                             isDisabled={isLoading}
+                            // runs on effect update
+                            onAuto={useCallback(() => changeRoute(delayedQuery), [delayedQuery, changeRoute])}
                             w={["full", null, "auto"]}
                         />
                     </Stack>
@@ -180,9 +249,10 @@ const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, transl
                         value={translation ?? ""}
                         readOnly={true}
                         lang={transLang}
-                        audio={audio?.target}
+                        audio={audio.translation}
                         canCopy={true}
                         isLoading={isLoading}
+                        pronunciation={pronunciation.translation}
                     />
                 </Stack>
             </VStack>
@@ -201,10 +271,12 @@ export const getStaticPaths: GetStaticPaths = async () => ({
     fallback: true
 });
 
-export const getStaticProps: GetStaticProps = async ({ params }) => {
+export const getStaticProps: GetStaticProps<Props> = async ({ params }) => {
     if (!params?.slug || !Array.isArray(params.slug))
         return {
-            props: { home: true }
+            props: {
+                type: ResponseType.HOME
+            }
         };
 
     const { source, target, query } = extractSlug(params.slug);
@@ -220,35 +292,52 @@ export const getStaticProps: GetStaticProps = async ({ params }) => {
                 destination: `/${source ?? "auto"}/${target ?? "en"}/${query}`,
                 permanent: true
             }
-        }
+        };
 
-    if (!isValid(source) || !isValid(target))
+    if (!isValidCode(source, LanguageType.SOURCE) || !isValidCode(target, LanguageType.TARGET))
         return {
             notFound: true
         };
 
-    const textScrape = await googleScrape(source, target, query);
+    const initial = { source, target, query };
 
-    const [sourceAudio, targetAudio] = await Promise.all([
-        textToSpeechScrape(source, query),
-        "translationRes" in textScrape
-            ? textToSpeechScrape(target, textScrape.translationRes)
-            : null
+    const translation = await getTranslationText(source, target, query);
+
+    if (!translation)
+        return {
+            props: {
+                type: ResponseType.ERROR,
+                errorMsg: "An error occurred while retrieving the translation",
+                initial
+            },
+            revalidate: 1
+        };
+
+    const info = await getTranslationInfo(source, target, query);
+
+    const audioSource = source === "auto" && info?.detectedSource
+        ? info.detectedSource
+        : source;
+    const parsedAudioSource = replaceExceptedCode(LanguageType.TARGET, audioSource);
+
+    const [audioQuery, audioTranslation] = await Promise.all([
+        getAudio(parsedAudioSource, query),
+        getAudio(target, translation)
     ]);
 
+    const audio = {
+        query: audioQuery,
+        translation: audioTranslation
+    };
+
     return {
         props: {
-            ...textScrape,
-            audio: {
-                source: sourceAudio,
-                target: targetAudio
-            },
-            initial: {
-                source, target, query
-            }
+            type: ResponseType.SUCCESS,
+            translation,
+            info,
+            audio,
+            initial
         },
-        revalidate: !("errorMsg" in textScrape)
-            ? 2 * 30 * 24 * 60 * 60 // 2 months
-            : 1
+        revalidate: 2 * 30 * 24 * 60 * 60 // 2 months
     };
 };

pages/api/graphql.ts

@@ -1,8 +1,16 @@
 import { ApolloServer, gql, IResolvers, ApolloError, UserInputError } from "apollo-server-micro";
 import { NextApiHandler } from "next";
 import NextCors from "nextjs-cors";
-import { googleScrape, textToSpeechScrape } from "@utils/translate";
-import { retrieveFromType, getName, isValid } from "@utils/language";
+import {
+    getTranslationInfo,
+    getTranslationText,
+    getAudio,
+    replaceExceptedCode,
+    isValidCode,
+    LanguageType,
+    languageList,
+    LangCode
+} from "lingva-scraper";
 
 export const typeDefs = gql`
     enum LangType {
@@ -11,14 +19,32 @@ export const typeDefs = gql`
     }
     type Query {
         translation(source: String="auto" target: String="en" query: String!): Translation!
-        audio(lang: String! query: String!): Entry!
+        audio(lang: String! query: String!): AudioEntry!
         languages(type: LangType): [Language]!
     }
     type Translation {
-        source: Entry!
-        target: Entry!
+        source: SourceEntry!
+        target: TargetEntry!
     }
-    type Entry {
+    type SourceEntry {
+        lang: Language!
+        text: String!
+        audio: [Int]!
+        detected: Language
+        typo: String
+        pronunciation: String
+        definitions: [DefinitionsGroup]
+        examples: [String]
+        similar: [String]
+    }
+    type TargetEntry {
+        lang: Language!
+        text: String!
+        audio: [Int]!
+        pronunciation: String
+        extraTranslations: [ExtraTranslationsGroup]
+    }
+    type AudioEntry {
         lang: Language!
         text: String!
         audio: [Int]!
@@ -27,34 +53,70 @@ export const typeDefs = gql`
         code: String!
         name: String!
     }
+    type DefinitionsGroup {
+        type: String!
+        list: [DefinitionList]!
+    }
+    type DefinitionList {
+        definition: String!
+        example: String!
+        field: String
+        synonyms: [String]
+    }
+    type ExtraTranslationsGroup {
+        type: String!
+        list: [ExtraTranslationList]!
+    }
+    type ExtraTranslationList {
+        word: String!
+        article: String
+        frequency: Int!
+        meanings: [String]
+    }
 `;
 
 export const resolvers: IResolvers = {
     Query: {
-        translation(_, args) {
+        async translation(_, args) {
             const { source, target, query } = args;
 
-            if (!isValid(source) || !isValid(target))
+            if (!isValidCode(source, LanguageType.SOURCE) || !isValidCode(target, LanguageType.TARGET))
                 throw new UserInputError("Invalid language code");
 
+            const translation = await getTranslationText(source, target, query);
+            if (!translation)
+                throw new ApolloError("An error occurred while retrieving the translation");
+
+            const info = await getTranslationInfo(source, target, query);
             return {
                 source: {
                     lang: {
                         code: source
                     },
-                    text: query
+                    text: query,
+                    detected: info?.detectedSource && {
+                        code: info.detectedSource
+                    },
+                    typo: info?.typo,
+                    pronunciation: info?.pronunciation.query,
+                    definitions: info?.definitions,
+                    examples: info?.examples,
+                    similar: info?.similar
                 },
                 target: {
                     lang: {
                         code: target
-                    }
+                    },
+                    text: translation,
+                    pronunciation: info?.pronunciation.translation,
+                    extraTranslations: info?.extraTranslations
                 }
             };
         },
         audio(_, args) {
             const { lang, query } = args;
 
-            if (!isValid(lang))
+            if (!isValidCode(lang))
                 throw new UserInputError("Invalid language code");
 
             return {
@@ -66,36 +128,28 @@ export const resolvers: IResolvers = {
         },
         languages(_, args) {
             const { type } = args;
-            const langEntries = retrieveFromType(type?.toLocaleLowerCase());
+            const lowerType = type?.toLocaleLowerCase() as typeof LanguageType[keyof typeof LanguageType] | undefined;
+            const langEntries = Object.entries(languageList[lowerType ?? "all"]);
             return langEntries.map(([code, name]) => ({ code, name }));
         }
     },
-    Translation: {
-        async target(parent) {
-            const { source, target } = parent;
-            const textScrape = await googleScrape(source.lang.code, target.lang.code, source.text);
-
-            if ("errorMsg" in textScrape)
-                throw new ApolloError(textScrape.errorMsg);
-            return {
-                lang: target.lang,
-                text: textScrape.translationRes
-            };
+    ...(["SourceEntry", "TargetEntry", "AudioEntry"].reduce((acc, key) => ({
+        ...acc,
+        [key]: {
+            async audio(parent) {
+                const { lang, text } = parent;
+                const parsedLang = replaceExceptedCode(LanguageType.TARGET, lang.code);
+                const audio = await getAudio(parsedLang, text);
+                if (!audio)
+                    throw new ApolloError("An error occurred while retrieving the audio");
+                return audio;
+            }
         }
-    },
-    Entry: {
-        async audio(parent) {
-            const { lang, text } = parent;
-            const audio = await textToSpeechScrape(lang.code, text);
-            if (!audio)
-                throw new ApolloError("An error occurred while retrieving the audio");
-            return audio;
-        }
-    },
+    }), {} as IResolvers)),
     Language: {
         name(parent) {
             const { code, name } = parent;
-            return name || getName(code);
+            return name || languageList.all[code as LangCode];
         }
     }
 };

pages/api/v1/[[...slug]].ts

@@ -1,10 +1,10 @@
 import { NextApiHandler } from "next";
 import NextCors from "nextjs-cors";
-import { googleScrape, textToSpeechScrape } from "@utils/translate";
-import { isValid } from "@utils/language";
+import { getTranslationInfo, getTranslationText, getAudio, isValidCode, LanguageType, TranslationInfo } from "lingva-scraper";
 
 type Data = {
     translation: string,
+    info?: TranslationInfo
 } | {
     audio: number[]
 } | {
@@ -35,24 +35,29 @@ const handler: NextApiHandler<Data> = async (req, res) => {
 
     const [source, target, query] = slug;
 
-    if (!isValid(target))
+    if (!isValidCode(target, LanguageType.TARGET))
         return res.status(400).json({ error: "Invalid target language" });
 
     if (source === "audio") {
-        const audio = await textToSpeechScrape(target, query);
+        const audio = await getAudio(target, query);
         return audio
             ? res.status(200).json({ audio })
             : res.status(500).json({ error: "An error occurred while retrieving the audio" });
     }
 
-    if (!isValid(source))
+    if (!isValidCode(source, LanguageType.SOURCE))
         return res.status(400).json({ error: "Invalid source language" });
 
-    const textScrape = await googleScrape(source, target, query);
+    const translation = await getTranslationText(source, target, query);
 
-    if ("errorMsg" in textScrape)
-        return res.status(500).json({ error: textScrape.errorMsg });
-    res.status(200).json({ translation: textScrape.translationRes });
+    if (!translation)
+        return res.status(500).json({ error: "An error occurred while retrieving the translation" });
+
+    const info = await getTranslationInfo(source, target, query);
+
+    return info
+        ? res.status(200).json({ translation, info })
+        : res.status(200).json({ translation });
 }
 
 export default handler;

pages/api/v1/languages/[[...slug]].ts

@@ -1,6 +1,6 @@
 import { NextApiHandler } from "next";
 import NextCors from "nextjs-cors";
-import { retrieveFromType, LangCode } from "@utils/language";
+import { languageList, LangCode } from "lingva-scraper";
 
 type Data = {
     languages: {
@@ -38,7 +38,7 @@ const handler: NextApiHandler<Data> = async (req, res) => {
     if (type !== undefined && type !== "source" && type !== "target")
         return res.status(400).json({ error: "Type should be 'source', 'target' or empty" });
 
-    const langEntries = retrieveFromType(type);
+    const langEntries = Object.entries(languageList[type ?? "all"]) as [LangCode, string][];
     const languages = langEntries.map(([code, name]) => ({ code, name }));
 
     res.status(200).json({ languages });