Skip to content

Commit 1260aec

Browse files
SuaYooSuaYoo
authored
fix: Crawler proxy selection fixes (#2280)
- Hides proxy form control if there are no proxy servers available - Fixes org default proxy value not being saved
1 parent d6189ee commit 1260aec

File tree

8 files changed

+165
-107
lines changed

8 files changed

+165
-107
lines changed

‎frontend/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"@formatjs/intl-durationformat": "^0.6.4",
99
"@formatjs/intl-localematcher": "^0.5.9",
1010
"@ianvs/prettier-plugin-sort-imports": "^4.2.1",
11+
"@lit/context": "^1.1.3",
1112
"@lit/localize": "^0.12.1",
1213
"@lit/task": "^1.0.0",
1314
"@novnc/novnc": "^1.4.0-beta",

‎frontend/src/components/ui/select-crawler-proxy.ts

+40-75
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import { localized, msg } from "@lit/localize";
2-
import { type SlSelect } from "@shoelace-style/shoelace";
2+
import type { SlSelect } from "@shoelace-style/shoelace";
33
import { html } from "lit";
44
import { customElement, property, state } from "lit/decorators.js";
5+
import { ifDefined } from "lit/directives/if-defined.js";
56

6-
import type { ProxiesAPIResponse, Proxy } from "@/pages/org/types";
7-
import LiteElement from "@/utils/LiteElement";
7+
import { BtrixElement } from "@/classes/BtrixElement";
8+
import type { Proxy } from "@/pages/org/types";
89

910
type SelectCrawlerProxyChangeDetail = {
1011
value: string | null;
@@ -26,30 +27,40 @@ export type SelectCrawlerProxyUpdateEvent =
2627
* Usage example:
2728
* ```ts
2829
* <btrix-select-crawler-proxy
29-
* orgId=${orgId}
30-
* on-change=${({value}) => selectedcrawlerProxy = value}
30+
* .proxyServers=${proxyServers}
31+
* btrix-change=${({value}) => selectedcrawlerProxy = value}
3132
* ></btrix-select-crawler-proxy>
3233
* ```
3334
*
34-
* @event on-change
35+
* @fires btrix-change
3536
*/
3637
@customElement("btrix-select-crawler-proxy")
3738
@localized()
38-
export class SelectCrawlerProxy extends LiteElement {
39+
export class SelectCrawlerProxy extends BtrixElement {
40+
@property({ type: String })
41+
defaultProxyId: string | null = null;
42+
43+
@property({ type: Array })
44+
proxyServers: Proxy[] = [];
45+
3946
@property({ type: String })
4047
proxyId: string | null = null;
4148

49+
@property({ type: String })
50+
size?: SlSelect["size"];
51+
4252
@state()
4353
private selectedProxy?: Proxy;
4454

4555
@state()
4656
private defaultProxy?: Proxy;
4757

48-
@state()
49-
private allProxies?: Proxy[];
58+
public get value() {
59+
return this.selectedProxy?.id || "";
60+
}
5061

5162
protected firstUpdated() {
52-
void this.fetchOrgProxies();
63+
void this.initProxies();
5364
}
5465
// credit: https://dev.to/jorik/country-code-to-flag-emoji-a21
5566
private countryCodeToFlagEmoji(countryCode: String): String {
@@ -61,10 +72,6 @@ export class SelectCrawlerProxy extends LiteElement {
6172
}
6273

6374
render() {
64-
/*if (this.crawlerProxys && this.crawlerProxys.length < 2) {
65-
return html``;
66-
}*/
67-
6875
return html`
6976
<sl-select
7077
name="proxyId"
@@ -75,15 +82,12 @@ export class SelectCrawlerProxy extends LiteElement {
7582
: msg("No Proxy")}
7683
hoist
7784
clearable
85+
size=${ifDefined(this.size)}
7886
@sl-change=${this.onChange}
79-
@sl-focus=${() => {
80-
// Refetch to keep list up to date
81-
void this.fetchOrgProxies();
82-
}}
8387
@sl-hide=${this.stopProp}
8488
@sl-after-hide=${this.stopProp}
8589
>
86-
${this.allProxies?.map(
90+
${this.proxyServers.map(
8791
(server) =>
8892
html` <sl-option value=${server.id}>
8993
${server.country_code
@@ -121,7 +125,7 @@ export class SelectCrawlerProxy extends LiteElement {
121125
private onChange(e: Event) {
122126
this.stopProp(e);
123127

124-
this.selectedProxy = this.allProxies?.find(
128+
this.selectedProxy = this.proxyServers.find(
125129
({ id }) => id === (e.target as SlSelect).value,
126130
);
127131

@@ -130,71 +134,32 @@ export class SelectCrawlerProxy extends LiteElement {
130134
}
131135

132136
this.dispatchEvent(
133-
new CustomEvent<SelectCrawlerProxyChangeDetail>("on-change", {
137+
new CustomEvent<SelectCrawlerProxyChangeDetail>("btrix-change", {
134138
detail: {
135139
value: this.selectedProxy ? this.selectedProxy.id : null,
136140
},
137141
}),
138142
);
139143
}
140144

141-
/**
142-
* Fetch crawler proxies and update internal state
143-
*/
144-
private async fetchOrgProxies(): Promise<void> {
145-
try {
146-
const data = await this.getOrgProxies();
147-
const defaultProxyId = data.default_proxy_id;
148-
149-
this.allProxies = data.servers;
150-
151-
if (!this.defaultProxy) {
152-
this.defaultProxy = this.allProxies.find(
153-
({ id }) => id === defaultProxyId,
154-
);
155-
}
156-
157-
if (this.proxyId && !this.selectedProxy?.id) {
158-
this.selectedProxy = this.allProxies.find(
159-
({ id }) => id === this.proxyId,
160-
);
161-
}
162-
163-
if (!this.selectedProxy) {
164-
this.proxyId = null;
165-
this.dispatchEvent(
166-
new CustomEvent("on-change", {
167-
detail: {
168-
value: null,
169-
},
170-
}),
171-
);
172-
this.selectedProxy = this.allProxies.find(
173-
({ id }) => id === this.proxyId,
174-
);
175-
}
176-
177-
this.dispatchEvent(
178-
new CustomEvent<SelectCrawlerProxyUpdateDetail>("on-update", {
179-
detail: {
180-
show: this.allProxies.length > 1,
181-
},
182-
}),
145+
private async initProxies(): Promise<void> {
146+
const defaultProxyId = this.defaultProxyId;
147+
148+
if (!this.defaultProxy) {
149+
this.defaultProxy = this.proxyServers.find(
150+
({ id }) => id === defaultProxyId,
183151
);
184-
} catch (e) {
185-
this.notify({
186-
message: msg("Sorry, couldn't retrieve proxies at this time."),
187-
variant: "danger",
188-
icon: "exclamation-octagon",
189-
id: "proxy-retrieve-status",
190-
});
191152
}
192-
}
193153

194-
private async getOrgProxies(): Promise<ProxiesAPIResponse> {
195-
return this.apiFetch<ProxiesAPIResponse>(
196-
`/orgs/${this.orgId}/crawlconfigs/crawler-proxies`,
197-
);
154+
if (this.proxyId && !this.selectedProxy) {
155+
this.selectedProxy = this.proxyServers.find(
156+
({ id }) => id === this.proxyId,
157+
);
158+
}
159+
160+
if (!this.selectedProxy) {
161+
this.proxyId = null;
162+
}
198163
}
199164

200165
/**

‎frontend/src/context/org.ts

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import { createContext } from "@lit/context";
2+
3+
import type { ProxiesAPIResponse } from "@/types/crawler";
4+
5+
export type ProxiesContext = ProxiesAPIResponse | null;
6+
7+
export const proxiesContext = createContext<ProxiesContext>("proxies");

‎frontend/src/features/browser-profiles/new-browser-profile-dialog.ts

+23-8
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,29 @@
1+
import { consume } from "@lit/context";
12
import { localized, msg, str } from "@lit/localize";
23
import { type SlInput } from "@shoelace-style/shoelace";
4+
import { nothing } from "lit";
35
import {
46
customElement,
57
property,
68
query,
79
queryAsync,
810
state,
911
} from "lit/decorators.js";
12+
import { ifDefined } from "lit/directives/if-defined.js";
1013
import queryString from "query-string";
1114

1215
import type { Dialog } from "@/components/ui/dialog";
1316
import { type SelectCrawlerChangeEvent } from "@/components/ui/select-crawler";
1417
import { type SelectCrawlerProxyChangeEvent } from "@/components/ui/select-crawler-proxy";
18+
import { proxiesContext, type ProxiesContext } from "@/context/org";
1519
import LiteElement, { html } from "@/utils/LiteElement";
1620

1721
@localized()
1822
@customElement("btrix-new-browser-profile-dialog")
1923
export class NewBrowserProfileDialog extends LiteElement {
24+
@consume({ context: proxiesContext, subscribe: true })
25+
private readonly proxies?: ProxiesContext;
26+
2027
@property({ type: Boolean })
2128
open = false;
2229

@@ -83,14 +90,22 @@ export class NewBrowserProfileDialog extends LiteElement {
8390
(this.crawlerChannel = e.detail.value!)}
8491
></btrix-select-crawler>
8592
</div>
86-
<div class="mt-4">
87-
<btrix-select-crawler-proxy
88-
orgId=${this.orgId}
89-
.proxyId="${this.proxyId || ""}"
90-
@on-change=${(e: SelectCrawlerProxyChangeEvent) =>
91-
(this.proxyId = e.detail.value!)}
92-
></btrix-select-crawler-proxy>
93-
</div>
93+
${this.proxies?.servers.length
94+
? html`
95+
<div class="mt-4">
96+
<btrix-select-crawler-proxy
97+
defaultProxyId=${ifDefined(
98+
this.proxies.default_proxy_id ?? undefined,
99+
)}
100+
.proxyServers=${this.proxies.servers}
101+
.proxyId="${this.proxyId || ""}"
102+
@btrix-change=${(e: SelectCrawlerProxyChangeEvent) =>
103+
(this.proxyId = e.detail.value)}
104+
></btrix-select-crawler-proxy>
105+
</div>
106+
`
107+
: nothing}
108+
94109
<input class="invisible size-0" type="submit" />
95110
</form>
96111
<div slot="footer" class="flex justify-between">

‎frontend/src/features/crawl-workflows/workflow-editor.ts

+23-11
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { consume } from "@lit/context";
12
import { localized, msg, str } from "@lit/localize";
23
import type {
34
SlChangeEvent,
@@ -43,6 +44,7 @@ import type { SelectCrawlerProxyChangeEvent } from "@/components/ui/select-crawl
4344
import type { Tab } from "@/components/ui/tab-list";
4445
import type { TagInputEvent, TagsChangeEvent } from "@/components/ui/tag-input";
4546
import type { TimeInputChangeEvent } from "@/components/ui/time-input";
47+
import { proxiesContext, type ProxiesContext } from "@/context/org";
4648
import { type SelectBrowserProfileChangeEvent } from "@/features/browser-profiles/select-browser-profile";
4749
import type { CollectionsChangeEvent } from "@/features/collections/collections-add";
4850
import type { QueueExclusionTable } from "@/features/crawl-workflows/queue-exclusion-table";
@@ -188,6 +190,9 @@ type CrawlConfigResponse = {
188190
@localized()
189191
@customElement("btrix-workflow-editor")
190192
export class WorkflowEditor extends BtrixElement {
193+
@consume({ context: proxiesContext, subscribe: true })
194+
private readonly proxies?: ProxiesContext;
195+
191196
@property({ type: String })
192197
configId?: string;
193198

@@ -1329,17 +1334,24 @@ https://archiveweb.page/images/${"logo.svg"}`}
13291334
></btrix-select-browser-profile>
13301335
`)}
13311336
${this.renderHelpTextCol(infoTextStrings["browserProfile"])}
1332-
${inputCol(html`
1333-
<btrix-select-crawler-proxy
1334-
orgId=${this.orgId}
1335-
.proxyId="${this.formState.proxyId || ""}"
1336-
@on-change=${(e: SelectCrawlerProxyChangeEvent) =>
1337-
this.updateFormState({
1338-
proxyId: e.detail.value,
1339-
})}
1340-
></btrix-select-crawler-proxy>
1341-
`)}
1342-
${this.renderHelpTextCol(infoTextStrings["proxyId"])}
1337+
${this.proxies?.servers.length
1338+
? [
1339+
inputCol(html`
1340+
<btrix-select-crawler-proxy
1341+
defaultProxyId=${ifDefined(
1342+
this.proxies.default_proxy_id ?? undefined,
1343+
)}
1344+
.proxyServers=${this.proxies.servers}
1345+
.proxyId="${this.formState.proxyId || ""}"
1346+
@btrix-change=${(e: SelectCrawlerProxyChangeEvent) =>
1347+
this.updateFormState({
1348+
proxyId: e.detail.value,
1349+
})}
1350+
></btrix-select-crawler-proxy>
1351+
`),
1352+
this.renderHelpTextCol(infoTextStrings["proxyId"]),
1353+
]
1354+
: nothing}
13431355
${inputCol(html`
13441356
<sl-radio-group
13451357
name="scale"

0 commit comments

Comments
 (0)