diff --git a/web/.env b/web/.env
index 96c129e..d6ec3a1 100644
--- a/web/.env
+++ b/web/.env
@@ -1,3 +1,7 @@
-# VITE_SONISORI_API_URL=https://api.sonisori.site
-VITE_SONISORI_API_URL=/
-VITE_SONISORI_AI_API_URL=http://127.0.0.1:5002
\ No newline at end of file
+# VITE_SONISORI_BFF_API_URL=https://api.sonisori.site
+VITE_SONISORI_BFF_API_URL=/bff
+
+# VITE_SONISORI_AI_REST_URL=https://ai.sonisori.site
+VITE_SONISORI_AI_REST_URL=/ai-rest
+
+VITE_SONISORI_AI_SOCKET_URL=wss://ai.sonisori.site
\ No newline at end of file
diff --git a/web/src/service/util/api.ts b/web/src/service/util/api.ts
index 86bffe4..bbe10db 100644
--- a/web/src/service/util/api.ts
+++ b/web/src/service/util/api.ts
@@ -1,14 +1,14 @@
import ky from "ky";
export const client = ky.create({
- prefixUrl: import.meta.env.VITE_SONISORI_API_URL,
+ prefixUrl: import.meta.env.VITE_SONISORI_BFF_API_URL,
throwHttpErrors: true,
hooks: {
afterResponse: [
async (request, options, response) => {
if (response.status === 401) {
await ky
- .create({ prefixUrl: import.meta.env.VITE_SONISORI_API_URL })
+ .create({ prefixUrl: import.meta.env.VITE_SONISORI_BFF_API_URL })
.get("api/reissue");
return ky(request, options);
}
diff --git a/web/src/ui/component/domain/SignDetector.tsx b/web/src/ui/component/domain/SignDetector.tsx
index b969642..51a6878 100644
--- a/web/src/ui/component/domain/SignDetector.tsx
+++ b/web/src/ui/component/domain/SignDetector.tsx
@@ -5,6 +5,7 @@ import {
} from "@mediapipe/tasks-vision";
import { createEventListener } from "@solid-primitives/event-listener";
import { createPresence } from "@solid-primitives/presence";
+import { throttle } from "@solid-primitives/scheduled";
import ky from "ky";
import { io, Socket } from "socket.io-client";
import {
@@ -50,7 +51,7 @@ export const createSentence = async (sign: {
}) => {
const phrase = await ky
.post(
- `${import.meta.env.VITE_SONISORI_AI_API_URL}${SIGN_PHRASE_TYPE_API_ENDPOINT_MAP[sign.signPhraseType]}`,
+ `${import.meta.env.VITE_SONISORI_AI_REST_URL}${SIGN_PHRASE_TYPE_API_ENDPOINT_MAP[sign.signPhraseType]}`,
{
json: { prediction: sign.words },
},
@@ -100,6 +101,10 @@ const SignDetectorBody = (props: {
let socket: Socket;
const task = new Task();
+ const send = throttle((landmarks: NormalizedLandmark[][]) => {
+ socket.emit("predict", landmarks);
+ }, 200);
+
const streamMedia = async () => {
if (!navigator.mediaDevices?.getDisplayMedia) {
throw new Error("카메라를 사용할 수 없는 디바이스입니다.");
@@ -149,7 +154,7 @@ const SignDetectorBody = (props: {
const { landmarks } = handLandmarker.detectForVideo(videoRef!, time);
drawLandmarks(landmarks);
- socket.emit("predict", landmarks);
+ send(landmarks);
if (typeof animationFrame == "number") {
animationFrame = requestAnimationFrame(predictMedia);
@@ -158,7 +163,7 @@ const SignDetectorBody = (props: {
const initialize = async () => {
try {
- socket = io(import.meta.env.VITE_SONISORI_AI_API_URL, {
+ socket = io(import.meta.env.VITE_SONISORI_AI_SOCKET_URL, {
transports: ["websocket"],
});
socket.on("prediction_result", (data: { appended: string }) => {
@@ -185,6 +190,7 @@ const SignDetectorBody = (props: {
animationFrame = null;
}
handLandmarker.close();
+ send.clear();
socket.disconnect();
stream?.getTracks().forEach((track) => track.stop());
setLoaded(false);
diff --git a/web/src/ui/screen/Dictionary.tsx b/web/src/ui/screen/Dictionary.tsx
index 0615f9a..d11032b 100644
--- a/web/src/ui/screen/Dictionary.tsx
+++ b/web/src/ui/screen/Dictionary.tsx
@@ -1,45 +1,79 @@
-import { useNavigate } from "@solidjs/router";
+import { useNavigate, useParams } from "@solidjs/router";
+import { createResource, For, Show } from "solid-js";
+import { client } from "../../service/util/api";
import { ScrollArea } from "../component/base/ScrollAreaV2";
import { SignDetector } from "../component/domain/SignDetector";
export const Dictionary = () => {
const navigate = useNavigate();
- const word = {
- word: "사과",
- videoUrl: "https://www.youtube.com/watch?v=9bZkp7q19f0",
- };
+ const { id } = useParams();
+ const [data] = createResource(() =>
+ client.get(`api/words/${id}`).json<{
+ description: string;
+ resources: {
+ resourceType: "image" | "video";
+ resourceUrl: string;
+ }[];
+ word: string;
+ }>(),
+ );
return (
navigate(-1)} open signPhraseType="평서문" />
-
-
-
-
{word.word}
- {/* video placeholder */}
-
-
-
-
-
수형 사진
-
-
![]()
+
+ {(data) => (
+
+
+
+
+ {data().word}
+
+ {/* video placeholder */}
+
+
+
수형 사진
+
+
resource.resourceType === "image",
+ )}
+ >
+ {(resource) => (
+
+ )}
+
+
+
수형 설명
+
{data().description}
+
- 수형 설명
-
- 오른 손바닥으로 주먹을 쥔 왼 팔을 쓸어내린 다음, 두 주먹을 쥐고
- 바닥이 아래로 향하게하여 가슴 앞에서 아래로 내린다.
-
-
-
-
+
+ )}
+
);
};
diff --git a/web/src/ui/screen/Learning.tsx b/web/src/ui/screen/Learning.tsx
index fc9e439..88d49ec 100644
--- a/web/src/ui/screen/Learning.tsx
+++ b/web/src/ui/screen/Learning.tsx
@@ -1,4 +1,5 @@
import { useNavigate, useParams } from "@solidjs/router";
+import ky from "ky";
import {
createEffect,
createResource,
@@ -9,17 +10,16 @@ import {
} from "solid-js";
import { createStore } from "solid-js/store";
-import { SignPhraseType } from "../../service/type/phrase";
import { client } from "../../service/util/api";
import { cn } from "../../service/util/cn";
import { Button } from "../component/base/Button";
import { Progress } from "../component/base/Progress";
import { SignDetector } from "../component/domain/SignDetector";
-enum History {
- CORRECT,
- INCORRECT,
- SKIP,
+enum Evaluated {
+ CORRECT = 1,
+ INCORRECT = 0,
+ UNKNOWN = -1,
}
const ProgressStatstics = (props: {
@@ -82,20 +82,23 @@ const TimerStatstics = (props: {
);
};
-const checkSentence = async (
- sentence: string,
- sign: { phraseType: SignPhraseType; words: string[] },
-) => {
- /**
- * @todo ai 서버 마이그레이션 후 로직 수정
- */
- return sentence === sign.words.join(" ") || sentence.includes(".");
+const checkSentence = async (quizIndex: number, sign: { words: string[] }) => {
+ const phrase = await ky
+ .post(`${import.meta.env.VITE_SONISORI_AI_REST_URL}/evaluateMeaning`, {
+ json: { prediction: sign.words, quizIndex },
+ })
+ .json()
+ .then(
+ (response) => response,
+ () => Evaluated.UNKNOWN,
+ );
+ return phrase;
};
export const Learning = () => {
const [quiz, setQuiz] = createStore({
index: 0,
- history: [] as History[],
+ history: [] as Evaluated[],
done: false,
});
@@ -114,7 +117,7 @@ export const Learning = () => {
quiz.history.length === 0
? 100
: Math.round(
- (quiz.history.filter((v) => v === History.CORRECT).length /
+ (quiz.history.filter((v) => v === Evaluated.CORRECT).length /
quiz.history.length) *
100,
);
@@ -129,25 +132,19 @@ export const Learning = () => {
onDone={(words) => {
if (loading()) return;
setLoading(true);
- checkSentence(data()[quiz.index].sentence, {
- phraseType: "평서문",
- words,
- })
+ checkSentence(data()[quiz.index].quizId, { words })
.then((result) => {
const done = quiz.index + 1 === data().length;
setQuiz((prev) => ({
index: done ? prev.index : prev.index + 1,
- history: [
- ...prev.history,
- result ? History.CORRECT : History.INCORRECT,
- ],
+ history: [...prev.history, result],
done,
}));
if (done) {
return client.post(`api/topics/${params.id}/result`, {
json: {
correctCount: quiz.history.filter(
- (v) => v === History.CORRECT,
+ (v) => v === Evaluated.CORRECT,
).length,
},
});
diff --git a/web/src/ui/screen/SignIn.tsx b/web/src/ui/screen/SignIn.tsx
index 10d6fc9..02d53bc 100644
--- a/web/src/ui/screen/SignIn.tsx
+++ b/web/src/ui/screen/SignIn.tsx
@@ -93,7 +93,7 @@ export const SignIn = () => {