From 20294677d418e3de1434809af0b5b7b0ea42a687 Mon Sep 17 00:00:00 2001 From: Nutthapat Pongtanyavichai <59821765+Leomotors@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:12:55 +0700 Subject: [PATCH 1/8] temp commit from stash --- apps/api/src/course/course.resolver.ts | 4 +- apps/api/src/search/queries/course.ts | 51 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/apps/api/src/course/course.resolver.ts b/apps/api/src/course/course.resolver.ts index e3f6f6b10..afb757949 100644 --- a/apps/api/src/course/course.resolver.ts +++ b/apps/api/src/course/course.resolver.ts @@ -9,7 +9,7 @@ import { isTime } from '@api/util/functions' import { Course, CourseDocument, DayOfWeek, GenEdType, Semester, StudyProgram } from '@cgr/schema' -import { CourseGroupInput, FilterInput, Period } from '../graphql' +import { CourseGroupInput, FilterInput, GradingType, Period } from '../graphql' import { CourseService } from './course.service' export interface ICourseSearchDocument { @@ -29,6 +29,7 @@ export interface ICourseSearchDocument { export interface ICourseSearchFilter { keyword: string genEdTypes?: GenEdType[] + gradingTypes?: GradingType[] dayOfWeeks?: DayOfWeek[] periodRange: Period studyProgram: string @@ -92,6 +93,7 @@ export class CourseResolver { keyword: filter.keyword, genEdTypes: filter.genEdTypes, dayOfWeeks: filter.dayOfWeeks, + gradingTypes: filter.gradingTypes, periodRange: filter.periodRange, studyProgram: courseGroup.studyProgram, semester: courseGroup.semester, diff --git a/apps/api/src/search/queries/course.ts b/apps/api/src/search/queries/course.ts index 6541de673..000b80b93 100644 --- a/apps/api/src/search/queries/course.ts +++ b/apps/api/src/search/queries/course.ts @@ -53,6 +53,57 @@ export function buildCourseQuery(filter: ICourseSearchFilter): Record 0) { boolMust.push({ nested: { From 71942ec9f569e3a284e655b2ad6fca7a754bd104 Mon Sep 17 00:00:00 2001 From: Nutthapat Pongtanyavichai <59821765+Leomotors@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:44:12 +0700 Subject: [PATCH 2/8] chore: changeset --- .changeset/red-baboons-brush.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/red-baboons-brush.md diff --git a/.changeset/red-baboons-brush.md b/.changeset/red-baboons-brush.md new file mode 100644 index 000000000..6736cf89f --- /dev/null +++ b/.changeset/red-baboons-brush.md @@ -0,0 +1,5 @@ +--- +'api': minor +--- + +fix: implement s/u filter in search From 6fa42fac690109be2303816903aa8081d99c06c7 Mon Sep 17 00:00:00 2001 From: samithiwat Date: Tue, 31 Oct 2023 15:14:38 +0700 Subject: [PATCH 3/8] feat: add cli reindex and delete index --- apps/opensearch-cli/README.md | 39 +++++++++++++++ apps/opensearch-cli/cli/app.go | 87 ++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/apps/opensearch-cli/README.md b/apps/opensearch-cli/README.md index 435bf1c59..0787d005a 100644 --- a/apps/opensearch-cli/README.md +++ b/apps/opensearch-cli/README.md @@ -26,3 +26,42 @@ go run ./cli/app.go index create --index-name= --filename= ```shell go run ./cli/app.go index create --index-name=course-dev-1 --filename=dev/course-index.json ``` + +# Reindex + +```shell +go run ./cli/app.go index reindex --index-name= --dest= --debug= +``` + +| name | description | +| ---------- | ---------------------------------- | +| index name | name of index | +| dest | name of destination index | +| debug | enable debug mode (`true`/`false`) | + +## Example + +### Reindex from course-dev-1 to course-dev-2 + +```shell +go run ./cli/app.go index reindex --index-name=course-dev-1 --dest=course-dev-2 +``` + +# Delete Index + +```shell +go run ./cli/app.go index reindex --index-name= --dest= --debug= +``` + +| name | description | +|------------|------------------------------------| +| index name | name of index | +| debug | enable debug mode (`true`/`false`) | + +## Example + +### Delete index course-dev-1 + +```shell +go run ./cli/app.go index delete --index-name=course-dev-1 +``` diff --git a/apps/opensearch-cli/cli/app.go b/apps/opensearch-cli/cli/app.go index 1447e4810..95a7e4553 100644 --- a/apps/opensearch-cli/cli/app.go +++ b/apps/opensearch-cli/cli/app.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "flag" "fmt" "github.com/opensearch-project/opensearch-go/v2" @@ -18,6 +19,7 @@ import ( var ( fileName string indexName string + destName string isDebug bool indexCmd *flag.FlagSet ) @@ -28,6 +30,7 @@ func init() { indexCmd = flag.NewFlagSet("index", flag.ExitOnError) indexCmd.StringVar(&indexName, "index-name", "", "Index name") + indexCmd.StringVar(&destName, "dest", "", "Destination index name") indexCmd.StringVar(&fileName, "filename", "", "Index filename") indexCmd.BoolVar(&isDebug, "debug", false, "Enable the debug log") flag.Parse() @@ -74,6 +77,26 @@ func main() { fmt.Println(strings.Repeat("_", 65)) fmt.Printf("successfully create the index, %s \n", indexName) + case "reindex": + if err := handleReindex(client); err != nil { + logger.Fatal( + "Error while creating the index", + zap.Error(err), + ) + } + + fmt.Println(strings.Repeat("_", 65)) + fmt.Printf("successfully reindex from %s to %s \n", indexName, destName) + case "delete": + if err := handleDeleteIndex(client); err != nil { + logger.Fatal( + "Error while creating the index", + zap.Error(err), + ) + } + + fmt.Println(strings.Repeat("_", 65)) + fmt.Printf("successfully delete the index, %s \n", indexName) default: logger.Fatal("Invalid input (invalid command)") } @@ -146,3 +169,67 @@ func handleCreateIndex(client *opensearch.Client) error { return nil } + +type ReIndexBody struct { + Source struct { + Index string `json:"index"` + } `json:"source"` + + Dest struct { + Index string `json:"index"` + } `json:"dest"` +} + +func handleReindex(client *opensearch.Client) error { + if err := indexCmd.Parse(os.Args[3:]); err != nil { + return err + } + + req, err := json.Marshal(ReIndexBody{ + Source: struct { + Index string `json:"index"` + }{Index: indexName}, + + Dest: struct { + Index string `json:"index"` + }{Index: destName}, + }) + + if err != nil { + return err + } + + res, err := client.Reindex( + bytes.NewReader(req), + ) + + if err != nil { + return err + } + + if res.IsError() { + return errors.New(res.String()) + } + + return nil +} + +func handleDeleteIndex(client *opensearch.Client) error { + if err := indexCmd.Parse(os.Args[3:]); err != nil { + return err + } + + res, err := client.Indices.Delete( + []string{indexName}, + ) + + if err != nil { + return err + } + + if res.IsError() { + return errors.New(res.String()) + } + + return nil +} From 122cbdb59e013a8708ebdd3ba401bf3d8f37029a Mon Sep 17 00:00:00 2001 From: samithiwat Date: Tue, 31 Oct 2023 15:17:07 +0700 Subject: [PATCH 4/8] feat: create reindex guideline --- apps/opensearch-cli/docs/reindex.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 apps/opensearch-cli/docs/reindex.md diff --git a/apps/opensearch-cli/docs/reindex.md b/apps/opensearch-cli/docs/reindex.md new file mode 100644 index 000000000..41fc581de --- /dev/null +++ b/apps/opensearch-cli/docs/reindex.md @@ -0,0 +1,5 @@ +# Reindex Guideline + +1. Create new index (destination index) by using command `go run ./cli/app.go index create --index-name= --file-name=` +2. Run reindex command `go run ./cli/app.go index reindex --index-name= --dest=` +3. Delete the old one `go run ./cli/app.go index delete --index-name=` From 4d8271cbc67077e55160d0392ddbfa56026fcc8d Mon Sep 17 00:00:00 2001 From: samithiwat Date: Tue, 31 Oct 2023 17:04:06 +0700 Subject: [PATCH 5/8] chore: update courseDoc --- .../src/scraper/queue-consumer/queue-consumer.service.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts b/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts index eaf81721d..55dd60a95 100644 --- a/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts +++ b/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts @@ -26,6 +26,7 @@ interface CourseDoc { studyProgram: StudyProgram semester: Semester academicYear: string + creditHours: string } @Processor({ @@ -76,6 +77,7 @@ export class QueueConsumerService { genEdType: course.genEdType, semester: course.semester, studyProgram: course.studyProgram, + creditHours: course.creditHours, } as CourseDoc, doc_as_upsert: true, }, From 9cbc3b10d18949f355f3c7b73f53d88022ffa3c0 Mon Sep 17 00:00:00 2001 From: samithiwat Date: Tue, 31 Oct 2023 17:04:14 +0700 Subject: [PATCH 6/8] fix: invalid command --- apps/opensearch-cli/docs/reindex.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/opensearch-cli/docs/reindex.md b/apps/opensearch-cli/docs/reindex.md index 41fc581de..4d8d73f13 100644 --- a/apps/opensearch-cli/docs/reindex.md +++ b/apps/opensearch-cli/docs/reindex.md @@ -1,5 +1,5 @@ # Reindex Guideline -1. Create new index (destination index) by using command `go run ./cli/app.go index create --index-name= --file-name=` +1. Create new index (destination index) by using command `go run ./cli/app.go index create --index-name= --filename=` 2. Run reindex command `go run ./cli/app.go index reindex --index-name= --dest=` 3. Delete the old one `go run ./cli/app.go index delete --index-name=` From e2aaf97449d2ab9aa16f2d7b180f8bc704348b08 Mon Sep 17 00:00:00 2001 From: samithiwat Date: Tue, 31 Oct 2023 18:42:45 +0700 Subject: [PATCH 7/8] chore: update index and scraper --- apps/opensearch-cli/index/beta/course-index.json | 8 +++++++- apps/opensearch-cli/index/dev/course-index.json | 8 +++++++- apps/opensearch-cli/index/prod/course-index.json | 8 +++++++- .../src/scraper/queue-consumer/queue-consumer.service.ts | 6 ++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/apps/opensearch-cli/index/beta/course-index.json b/apps/opensearch-cli/index/beta/course-index.json index b319ac5d3..4f3586738 100644 --- a/apps/opensearch-cli/index/beta/course-index.json +++ b/apps/opensearch-cli/index/beta/course-index.json @@ -1,6 +1,6 @@ { "aliases": { - "course-beta": {} + "cgr-course-beta": {} }, "settings": { "index": { @@ -73,6 +73,9 @@ } } }, + "suggestions": { + "type": "completion" + }, "abbrName": { "type": "text", "analyzer": "courseabbranal" @@ -106,6 +109,9 @@ }, "academicYear": { "type": "keyword" + }, + "creditHours": { + "type": "keyword" } } } diff --git a/apps/opensearch-cli/index/dev/course-index.json b/apps/opensearch-cli/index/dev/course-index.json index 25851cb4b..5deefe74f 100644 --- a/apps/opensearch-cli/index/dev/course-index.json +++ b/apps/opensearch-cli/index/dev/course-index.json @@ -1,6 +1,6 @@ { "aliases": { - "course-dev": {} + "cgr-course-dev": {} }, "settings": { "index": { @@ -73,6 +73,9 @@ } } }, + "suggestions": { + "type": "completion" + }, "abbrName": { "type": "text", "analyzer": "courseabbranal" @@ -106,6 +109,9 @@ }, "academicYear": { "type": "keyword" + }, + "creditHours": { + "type": "keyword" } } } diff --git a/apps/opensearch-cli/index/prod/course-index.json b/apps/opensearch-cli/index/prod/course-index.json index 1fa3ec39b..a9a0bb661 100644 --- a/apps/opensearch-cli/index/prod/course-index.json +++ b/apps/opensearch-cli/index/prod/course-index.json @@ -1,6 +1,6 @@ { "aliases": { - "course-prod": {} + "cgr-course-prod": {} }, "settings": { "index": { @@ -73,6 +73,9 @@ } } }, + "suggestions": { + "type": "completion" + }, "abbrName": { "type": "text", "analyzer": "courseabbranal" @@ -106,6 +109,9 @@ }, "academicYear": { "type": "keyword" + }, + "creditHours": { + "type": "keyword" } } } diff --git a/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts b/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts index 55dd60a95..159393621 100644 --- a/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts +++ b/apps/reg-scraper/src/scraper/queue-consumer/queue-consumer.service.ts @@ -78,6 +78,12 @@ export class QueueConsumerService { semester: course.semester, studyProgram: course.studyProgram, creditHours: course.creditHours, + suggestions: [ + { + input: [course.courseNo, course.abbrName, course.courseNameEn, course.courseNameTh], + weight: 20, + }, + ], } as CourseDoc, doc_as_upsert: true, }, From a91781567dcbdeea1a62494b292ca1aab7b6343d Mon Sep 17 00:00:00 2001 From: samithiwat Date: Tue, 31 Oct 2023 18:44:38 +0700 Subject: [PATCH 8/8] chore: update query --- apps/api/src/search/queries/course.ts | 56 +++++---------------------- 1 file changed, 10 insertions(+), 46 deletions(-) diff --git a/apps/api/src/search/queries/course.ts b/apps/api/src/search/queries/course.ts index 000b80b93..7428e58c1 100644 --- a/apps/api/src/search/queries/course.ts +++ b/apps/api/src/search/queries/course.ts @@ -4,6 +4,8 @@ import { ICourseSearchFilter } from '@api/course/course.resolver' export function buildCourseQuery(filter: ICourseSearchFilter): Record { // create the base query from values that guarantee is not undefined const boolMust: Record[] = [] + const boolMustNot: Record[] = [] + const boolFilter: Record[] = [] if (filter.keyword) { boolMust.push({ @@ -56,50 +58,14 @@ export function buildCourseQuery(filter: ICourseSearchFilter): Record 0 && !(containsSU && containsLetter)) { + if (containsSU) { + boolFilter.push({ + wildcard: { creditHours: { value: '*(S/U)*' } }, }) - } else if (containsSU) { - boolMust.push({ - nested: { - path: 'rawData', - query: { - nested: { - path: 'rawData.creditHours', - query: { - bool: { - must: [ - { - wildcard: { - 'rawData.creditHours': '*S/U*', - }, - }, - ], - }, - }, - }, - }, - }, + } else if (containsLetter) { + boolMustNot.push({ + wildcard: { creditHours: { value: '*(S/U)*' } }, }) } } @@ -155,9 +121,7 @@ export function buildCourseQuery(filter: ICourseSearchFilter): Record 0) { - result['bool'] = { must: boolMust } - } + result['bool'] = { must: boolMust, must_not: boolMustNot, filter: boolFilter } return result }