CloudHost/LingvAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Text to Speech (#2)

Browse Source 
* Initial TTS scrapping implemented

* Audio & copy buttons added

* TTS langs mapping fix

* Webkit audio api fix

* Added TTS-related testing

* Last tweaks
This commit is contained in:
David
2021-03-25 16:48:46 +01:00
committed by GitHub
parent 034355b640
commit 7288e9ace7
11 changed files with 302 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
components/TranslationArea.tsx
View File

@@ -1,24 +1,65 @@
import { FC, ChangeEvent } from "react";
import { Textarea, useBreakpointValue } from "@chakra-ui/react";
import { Box, HStack, Textarea, IconButton, Tooltip, useBreakpointValue, useClipboard } from "@chakra-ui/react";
import { FaCopy, FaCheck, FaPlay, FaStop } from "react-icons/fa";
import { useAudioFromBuffer } from "../hooks";
type Props = {
value: string,
onChange?: (e: ChangeEvent<HTMLTextAreaElement>) => void,
readOnly?: true
readOnly?: true,
audio?: number[],
canCopy?: boolean,
[key: string]: any
};
const TranslationArea: FC<Props> = ({ value, onChange, readOnly, ...props }) => (
<Textarea
value={value}
onChange={onChange}
readOnly={readOnly}
dir="auto"
resize="none"
rows={useBreakpointValue([6, null, 12]) ?? undefined}
size="lg"
{...props}
/>
);
const TranslationArea: FC<Props> = ({ value, onChange, readOnly, audio, canCopy, ...props }) => {
const { hasCopied, onCopy } = useClipboard(value);
const { audioExists, isAudioPlaying, onAudioClick } = useAudioFromBuffer(audio);
return (
<Box
position="relative"
w="full"
>
<Textarea
value={value}
onChange={onChange}
readOnly={readOnly}
dir="auto"
resize="none"
rows={useBreakpointValue([6, null, 12]) ?? undefined}
size="lg"
{...props}
/>
<HStack
position="absolute"
bottom={4}
right={4}
>
{canCopy && (
<Tooltip label={hasCopied ? "Copied!" : "Copy to clipboard"}>
<IconButton
aria-label="Copy to clipboard"
icon={hasCopied ? <FaCheck /> : <FaCopy />}
onClick={onCopy}
colorScheme="lingva"
variant="ghost"
disabled={!value}
/>
</Tooltip>
)}
<Tooltip label={isAudioPlaying ? "Stop audio" : "Play audio"}>
<IconButton
aria-label={isAudioPlaying ? "Stop audio" : "Play audio"}
icon={isAudioPlaying ? <FaStop /> : <FaPlay />}
onClick={onAudioClick}
colorScheme="lingva"
variant="ghost"
disabled={!audioExists}
/>
</Tooltip>
</HStack>
</Box>
);
};
export default TranslationArea;

49
cypress/integration/app.spec.ts
View File

@@ -58,14 +58,7 @@ it("switches first loaded page and back and forth on language change", () => {
.should("include", `/auto/en/${encodeURIComponent(query)}`);
});
it("doesn't switch initial page on language change", () => {
cy.findByRole("combobox", { name: /source language/i })
.select("eo")
.url()
.should("not.include", "/eo");
});
it("language switching button is disable on 'auto', but enables when other", () => {
it("language switching button is disabled on 'auto', but enables when other", () => {
cy.findByRole("button", { name: /switch languages/i })
.as("btnSwitch")
.should("be.disabled");
@@ -78,10 +71,38 @@ it("language switching button is disable on 'auto', but enables when other", ()
cy.findByRole("combobox", { name: /target language/i })
.should("have.value", "eo")
.get("@source")
.should("have.value", "en");
.should("have.value", "en")
.url()
.should("not.include", "/en")
.should("not.include", "/eo");
});
it("toggles color mode on button click", () => {
it("loads & plays audio correctly", () => {
const query = faker.lorem.words(5);
cy.visit(`/la/en/${query}`);
const play = "Play audio";
const stop = "Stop audio";
cy.findAllByRole("button", { name: play })
.should("be.enabled")
.click({ multiple: true })
.should("have.attr", "aria-label", stop)
.click({ multiple: true })
.should("have.attr", "aria-label", play)
.click({ multiple: true })
.should("have.attr", "aria-label", play)
.click({ multiple: true })
.should("have.attr", "aria-label", stop);
});
it("skips to main & toggles color mode", () => {
cy.findByRole("link", { name: /skip to content/i })
.focus()
.click()
.url()
.should("include", "#main");
const white = "rgb(255, 255, 255)";
cy.get("body")
.should("have.css", "background-color", white);
@@ -95,11 +116,3 @@ it("toggles color mode on button click", () => {
.get("body")
.should("have.css", "background-color", white);
});
it("skips to main on 'skip link' click", () => {
cy.findByRole("link", { name: /skip to content/i })
.focus()
.click()
.url()
.should("include", "#main");
});

1
hooks/index.ts
View File

@@ -1 +1,2 @@
export { default as useToastOnLoad } from "./useToastOnLoad";
export { default as useAudioFromBuffer } from "./useAudioFromBuffer";

60
hooks/useAudioFromBuffer.ts Normal file
View File

@@ -0,0 +1,60 @@
import { useState, useEffect, useRef } from "react";
declare global {
interface Window {
webkitAudioContext: typeof AudioContext
}
}
const useAudioFromBuffer = (bufferArray?: number[]) => {
const audioCtxRef = useRef<AudioContext | null>(null);
const [audioSource, setAudioSource] = useState<AudioBufferSourceNode | null>(null);
const [audioBuffer, setAudioBuffer] = useState<AudioBuffer | null>(null);
useEffect(() => {
const AudioCtx = window.AudioContext || window.webkitAudioContext;
if (!AudioCtx)
return;
audioCtxRef.current = new AudioCtx();
return () => {
audioCtxRef.current?.close();
}
}, []);
useEffect(() => {
if (!bufferArray)
return setAudioBuffer(null);
audioCtxRef.current?.decodeAudioData(
new Uint8Array(bufferArray).buffer
).then(setAudioBuffer);
}, [bufferArray]);
const onAudioClick = () => {
if (!audioCtxRef.current)
return;
if (!audioSource) {
const source = audioCtxRef.current.createBufferSource();
source.buffer = audioBuffer;
source.connect(audioCtxRef.current.destination);
source.start();
source.onended = () => {
setAudioSource(null);
}
setAudioSource(source);
return;
}
audioSource.stop();
audioSource.disconnect(audioCtxRef.current.destination);
setAudioSource(null);
};
return {
audioExists: !!audioBuffer,
isAudioPlaying: !!audioSource,
onAudioClick
};
}
export default useAudioFromBuffer;

4
package.json
View File

@@ -22,7 +22,8 @@
"next-pwa": "^5.0.6",
"react": "17.0.1",
"react-dom": "17.0.1",
"react-icons": "^4.2.0"
"react-icons": "^4.2.0",
"user-agents": "^1.0.597"
},
"devDependencies": {
"@testing-library/cypress": "^7.0.4",
@@ -33,6 +34,7 @@
"@types/jest": "^26.0.20",
"@types/node": "^14.14.33",
"@types/react": "^17.0.3",
"@types/user-agents": "^1.0.0",
"@typescript-eslint/eslint-plugin": "^4.0.0",
"@typescript-eslint/parser": "^4.0.0",
"babel-eslint": "^10.0.0",

22
pages/[[...slug]].tsx
View File

@@ -5,11 +5,11 @@ import { Stack, VStack, HStack, IconButton } from "@chakra-ui/react";
import { FaExchangeAlt } from "react-icons/fa";
import { CustomError, Layout, LangSelect, TranslationArea } from "../components";
import { useToastOnLoad } from "../hooks";
import { googleScrape, extractSlug } from "../utils/translate";
import { googleScrape, extractSlug, textToSpeechScrape } from "../utils/translate";
import { retrieveFiltered, replaceBoth } from "../utils/language";
import langReducer, { Actions, initialState } from "../utils/reducer";
const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, translationRes, statusCode, errorMsg, initial }) => {
const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, translationRes, audio, statusCode, errorMsg, initial }) => {
const [{ source, target, query, delayedQuery, translation }, dispatch] = useReducer(langReducer, initialState);
const handleChange = (e: ChangeEvent<HTMLTextAreaElement | HTMLSelectElement>) => {
@@ -92,6 +92,7 @@ const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, transl
value={query}
onChange={handleChange}
lang={queryLang}
audio={audio?.source}
/>
<TranslationArea
id="translation"
@@ -100,6 +101,8 @@ const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ home, transl
value={translation ?? ""}
readOnly={true}
lang={transLang}
audio={audio?.target}
canCopy={true}
/>
</Stack>
</VStack>
@@ -139,16 +142,25 @@ export const getStaticProps: GetStaticProps = async ({ params }) => {
}
}
const scrapeRes = await googleScrape(source, target, query);
const textScrape = await googleScrape(source, target, query);
const [sourceAudio, targetAudio] = await Promise.all([
textToSpeechScrape(source, query),
textToSpeechScrape(target, textScrape.translationRes)
]);
return {
props: {
...scrapeRes,
...textScrape,
audio: {
source: sourceAudio,
target: targetAudio
},
initial: {
source, target, query
}
},
revalidate: !scrapeRes.errorMsg && !scrapeRes.statusCode
revalidate: !textScrape.errorMsg && !textScrape.statusCode
? 2 * 30 * 24 * 60 * 60 // 2 months
: 1
};

36
tests/pages/[[...slug]].test.tsx
View File

@@ -36,7 +36,7 @@ describe("getStaticProps", () => {
expect(await getStaticProps({ params: { slug } })).toMatchObject({ redirect: expect.any(Object) });
});
it("returns translation & initial values on 3 params", async () => {
it("returns translation, audio & initial values on 3 params", async () => {
const translationRes = faker.random.words();
resolveFetchWith(htmlRes(translationRes));
@@ -44,6 +44,10 @@ describe("getStaticProps", () => {
expect(await getStaticProps({ params: { slug } })).toStrictEqual({
props: {
translationRes,
audio: {
source: expect.any(Array),
target: expect.any(Array)
},
initial: {
source,
target,
@@ -56,6 +60,13 @@ describe("getStaticProps", () => {
});
describe("Page", () => {
const translationRes = faker.random.words();
const randomAudio = Array.from({ length: 10 }, () => faker.random.number(100));
const audio = {
source: randomAudio,
target: randomAudio
};
it("loads the layout correctly", async () => {
render(<Page home={true} />);
@@ -88,8 +99,7 @@ describe("Page", () => {
target: "es",
query: faker.random.words()
};
const translationRes = faker.random.words();
render(<Page translationRes={translationRes} initial={initial} />);
render(<Page translationRes={translationRes} audio={audio} initial={initial} />);
const source = screen.getByRole("combobox", { name: /source language/i });
expect(source).toHaveValue(initial.source);
@@ -107,8 +117,7 @@ describe("Page", () => {
target: "en",
query: faker.random.words()
};
const translationRes = faker.random.words();
render(<Page translationRes={translationRes} initial={initial} />);
render(<Page translationRes={translationRes} audio={audio} initial={initial} />);
const source = screen.getByRole("combobox", { name: /source language/i });
@@ -137,8 +146,7 @@ describe("Page", () => {
target: "ca",
query: faker.random.words()
};
const translationRes = faker.random.words();
render(<Page translationRes={translationRes} initial={initial} />);
render(<Page translationRes={translationRes} audio={audio} initial={initial} />);
const btnSwitch = screen.getByRole("button", { name: /switch languages/i });
userEvent.click(btnSwitch);
@@ -151,6 +159,20 @@ describe("Page", () => {
await waitFor(() => expect(Router.push).toHaveBeenCalledTimes(1));
});
it("loads audio & clipboard correctly", async () => {
const initial = {
source: "eo",
target: "zh",
query: faker.random.words()
};
render(<Page translationRes={translationRes} audio={audio} initial={initial} />);
const btnsAudio = screen.getAllByRole("button", { name: /play audio/i });
btnsAudio.forEach(btn => expect(btn).toBeVisible());
const btnCopy = screen.getByRole("button", { name: /copy to clipboard/i });
expect(btnCopy).toBeEnabled();
});
it("renders error page on status code", async () => {
const code = faker.random.number({ min: 400, max: 599 });
render(<Page statusCode={code} />);

20
tests/utils/translate.test.ts
View File

@@ -1,6 +1,6 @@
import { htmlRes, resolveFetchWith } from "../commonUtils";
import faker from "faker";
import { googleScrape, extractSlug } from "../../utils/translate";
import { googleScrape, extractSlug, textToSpeechScrape } from "../../utils/translate";
const source = faker.random.locale();
const target = faker.random.locale();
@@ -65,3 +65,21 @@ describe("extractSlug", () => {
expect(extractSlug(array)).toStrictEqual({});
});
});
describe("textToSpeechScrape", () => {
it("returns an array on successful request", async () => {
resolveFetchWith({ status: 200 });
expect(await textToSpeechScrape(target, query)).toEqual(expect.any(Array));
});
it("returns 'null' on request error", async () => {
const status = faker.random.number({ min: 400, max: 499 });
resolveFetchWith({ status });
expect(await textToSpeechScrape(target, query)).toBeNull();
});
it("returns 'null' on network error", async () => {
fetchMock.mockRejectOnce();
expect(await textToSpeechScrape(target, query)).toBeNull();
});
});

3
utils/languages.json
View File

@@ -123,7 +123,8 @@
"source": {},
"target": {
"zh": "zh-CN",
"zh_HANT": "zh-TW"
"zh_HANT": "zh-TW",
"auto": "en"
}
}
}

38
utils/translate.ts
View File

@@ -1,3 +1,4 @@
import UserAgent from "user-agents";
import cheerio from "cheerio";
import { replaceBoth } from "./language";
@@ -12,8 +13,15 @@ export async function googleScrape(
}> {
const parsed = replaceBoth("mapping", { source, target });
const res = await fetch(
`https://translate.google.com/m?sl=${parsed.source}&tl=${parsed.target}&q=${encodeURIComponent(query)}`
).catch(() => null);
`https://translate.google.com/m?sl=${parsed.source}&tl=${parsed.target}&q=${encodeURIComponent(query)}`,
{
headers: {
"User-Agent": new UserAgent().toString()
}
}
).catch(
() => null
);
if (!res)
return {
@@ -53,3 +61,29 @@ export function extractSlug(slug: string[]): {
return {};
}
}
export async function textToSpeechScrape(lang: string, text?: string) {
if (!text)
return null;
const { target: parsedLang } = replaceBoth("mapping", { source: "", target: lang });
const lastSpace = text.lastIndexOf(" ", 200);
const slicedText = text.slice(0, text.length > 200 && lastSpace !== -1 ? lastSpace : 200);
const res = await fetch(
`http://translate.google.com/translate_tts?tl=${parsedLang}&q=${encodeURIComponent(slicedText)}&textlen=${slicedText.length}&client=tw-ob`,
{
headers: {
"User-Agent": new UserAgent().toString()
}
}
).catch(
() => null
);
return res?.ok
? res.blob().then(blob => blob.arrayBuffer()).then(buffer => Array.from(new Uint8Array(buffer)))
: null;
}

49
yarn.lock
View File

@@ -2266,6 +2266,11 @@
dependencies:
source-map "^0.6.1"
"@types/user-agents@^1.0.0":
version "1.0.0"
resolved "https://registry.yarnpkg.com/@types/user-agents/-/user-agents-1.0.0.tgz#aeb7546f50cea28358a019a70ab3ce8827ecf937"
integrity sha512-oOSdQ9CULdFN2SJ9NNHvPrkP9aJ6oAz7BiiAeMC4vca+AzFqktlGdetMjlbCA0J3AemMn7ToU3U74VgrMecHtA==
"@types/warning@^3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@types/warning/-/warning-3.0.0.tgz#0d2501268ad8f9962b740d387c4654f5f8e23e52"
@@ -3875,6 +3880,11 @@ des.js@^1.0.0:
inherits "^2.0.1"
minimalistic-assert "^1.0.0"
detect-indent@~6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.0.0.tgz#0abd0f549f69fc6659a254fe96786186b6f528fd"
integrity sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA==
detect-newline@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651"
@@ -3906,6 +3916,11 @@ dir-glob@^3.0.1:
dependencies:
path-type "^4.0.0"
docopt@~0.6.2:
version "0.6.2"
resolved "https://registry.yarnpkg.com/docopt/-/docopt-0.6.2.tgz#b28e9e2220da5ec49f7ea5bb24a47787405eeb11"
integrity sha1-so6eIiDaXsSffqW7JKR3h0Be6xE=
doctrine@1.5.0:
version "1.5.0"
resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-1.5.0.tgz#379dce730f6166f76cefa4e6707a159b02c5a6fa"
@@ -3975,6 +3990,15 @@ domutils@^2.4.3, domutils@^2.4.4:
domelementtype "^2.0.1"
domhandler "^4.0.0"
dot-json@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/dot-json/-/dot-json-1.2.2.tgz#7d35abece4aa22aa75a761388953f98495401bcc"
integrity sha512-AKL+GsO4wSEU4LU+fAk/PqN4nQ6PB1vT3HpMiZous9xCzK5S0kh4DzfUY0EfU67jsIXLlu0ty71659N9Nmg+Tw==
dependencies:
detect-indent "~6.0.0"
docopt "~0.6.2"
underscore-keypath "~0.0.22"
ecc-jsbn@~0.1.1:
version "0.1.2"
resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -6221,6 +6245,11 @@ locate-path@^5.0.0:
dependencies:
p-locate "^4.1.0"
lodash.clonedeep@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
integrity sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=
lodash.debounce@^4.0.8:
version "4.0.8"
resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
@@ -8633,6 +8662,18 @@ unbox-primitive@^1.0.0:
has-symbols "^1.0.0"
which-boxed-primitive "^1.0.1"
underscore-keypath@~0.0.22:
version "0.0.22"
resolved "https://registry.yarnpkg.com/underscore-keypath/-/underscore-keypath-0.0.22.tgz#48a528392bb6efc424be1caa56da4b5faccf264d"
integrity sha1-SKUoOSu278QkvhyqVtpLX6zPJk0=
dependencies:
underscore "*"
underscore@*:
version "1.12.1"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.12.1.tgz#7bb8cc9b3d397e201cf8553336d262544ead829e"
integrity sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==
unicode-canonical-property-names-ecmascript@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz#2619800c4c825800efdd8343af7dd9933cbe2818"
@@ -8751,6 +8792,14 @@ use@^3.1.0:
resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f"
integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==
user-agents@^1.0.597:
version "1.0.597"
resolved "https://registry.yarnpkg.com/user-agents/-/user-agents-1.0.597.tgz#a61659191d99d6de303855dda78cbbded22e5355"
integrity sha512-BwG3FDnNEg9+LBaej6cUJlvL3ztk9+8SCiIYLkm9TXR+QgIH5CcRs4tPVz9CdV3niArmOa5sDCM69iv8bUnkZA==
dependencies:
dot-json "^1.2.2"
lodash.clonedeep "^4.5.0"
util-deprecate@^1.0.1, util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"