Skip to content
This repository has been archived by the owner on Sep 20, 2023. It is now read-only.

Commit

Permalink
Better script gen & fix recording stop listening
Browse files Browse the repository at this point in the history
  • Loading branch information
baptisteArno committed Mar 17, 2021
1 parent f9252db commit a01737e
Show file tree
Hide file tree
Showing 5 changed files with 30 additions and 13 deletions.
2 changes: 1 addition & 1 deletion public/content.js
Original file line number Diff line number Diff line change
Expand Up @@ -269,8 +269,8 @@ const stopSelectNode = () => {
for (const handler of handlers) {
document.removeEventListener("click", handler, { capture: true });
}
handlers = [];
stopClicksKeysRecording();
handlers = [];
};

const onRecordClick = function (stepIndex) {
Expand Down
2 changes: 1 addition & 1 deletion public/manifest.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "Tinking - Scrapping Tool",
"version": "0.1.0",
"version": "0.1.1",
"manifest_version": 2,
"description": "Extract data from any website without code, just clicks",
"icons": {
Expand Down
2 changes: 1 addition & 1 deletion src/StepItem/OptionItem.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ const SelectOption = ({
optionType?: OptionType;
onOptionChange: (val: OptionType) => void;
}) => (
<Menu>
<Menu defaultIsOpen={optionType === undefined}>
<MenuButton
size="sm"
as={Button}
Expand Down
2 changes: 1 addition & 1 deletion src/StepItem/StepItem.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ const SelectAction = ({
step: Step;
onActionChange: (val: StepAction) => void;
}) => (
<Menu>
<Menu defaultIsOpen={step.action === undefined}>
<MenuButton size="sm" as={Button} rightIcon={<ChevronDownIcon />}>
{step.action ?? "Select an action"}
</MenuButton>
Expand Down
35 changes: 26 additions & 9 deletions src/lib/scriptGenerator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ const parseSingleCommandFromStep = (
const element = document.querySelector("${step.selector}")
return element.src || null;
}
let ${variableName} = await page.evaluate(() => ${variableName}Eval);
let ${variableName} = await page.evaluate(${variableName}Eval);
if(${variableName} === null || ${variableName} === ""){
// The content could be dynamically loaded. Waiting a bit...
await page.waitForTimeout(4000)
Expand Down Expand Up @@ -341,9 +341,13 @@ const parseLoopFromStep = (step: Step) => {
await page.waitForSelector("${step.selector}")
let urls = []
urls = await page.evaluate(() => {
return [...document.querySelectorAll("${step.selector}")].map((node) => node.href);
return [...document.querySelectorAll("${
step.selector
}")].map((node) => node.href);
});
if(urls.length >= ${amountToExtract}){
if(${
amountToExtract === "" ? "false" : `urls.length >= ${amountToExtract}`
}){
urls = urls.slice(0, ${amountToExtract})
} else {
let i = 0
Expand All @@ -366,18 +370,30 @@ const parseLoopFromStep = (step: Step) => {
}catch{
break;
}
const firstLinkInNewPage = (
await page.$("${step.selector}")
).href;
if (!firstLinkInNewPage || firstLinkInNewPage === firstLinkInCurrentPage) {
let firstLinkInNewPage = await page.evaluate(() => {return document.querySelector("${
step.selector
}").href});
if (firstLinkInNewPage === firstLinkInCurrentPage) {
// There is some kind of loading state we need to wait for
await page.waitForTimeout(4000);
firstLinkInNewPage = await page.evaluate(() => {return document.querySelector("${
step.selector
}").href});
if (firstLinkInNewPage === firstLinkInCurrentPage) {
break;
}
}
const newUrls = await page.evaluate(() => {
return [...document.querySelectorAll("${step.selector}")].map(node => node.href);
return [...document.querySelectorAll("${
step.selector
}")].map(node => node.href);
})
urls = urls.concat(newUrls)
if (urls.length >= ${amountToExtract}) {
if (${
amountToExtract === ""
? "false"
: `urls.length >= ${amountToExtract}`
}) {
urls = urls.slice(0, ${amountToExtract})
break;
}
Expand Down Expand Up @@ -460,6 +476,7 @@ const parseLibrarySettings = (library: "puppeteer" | "playwright") => {
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--window-size=1300,1024"
],
});`;
}
Expand Down

0 comments on commit a01737e

Please sign in to comment.