/* * Usage: * * 1. Create config.json, and add these contents to it: * {"username": "", "password": "", "summary": "Import adverb from Wiktionary"} * 2. Run yarn add node-fetch wikidata-edit wikidata-sdk * 3. Run node */ const config = require("./config.json"); const searchLimit = 500; const dictApiUrl = "https://en.wiktionary.org/w/api.php?maxlag=5&action=query&list=categorymembers&cmtitle=Category%3AEnglish_adverbs&cmlimit=" + searchLimit + "&format=json"; const fetch = require("node-fetch"); const wdk = require("wikidata-sdk"); const fs = require("fs"); const { post } = require("wikidata-edit/lib/request.js")(config); const wdEdit = require("wikidata-edit")(config); /* { labels: {}, descriptions: {}, aliases: {}, claims: {}, sitelinks: {} } */ async function lexemeExists(search, language = "en") { const url = wdk.searchEntities({ search: search, language: language, limit: 1, format: "json", type: "lexeme" }) .replace("https://www.wikidata", "https://test.wikidata") + "&maxlag=1"; let result; try { result = await fetch(url); } catch (e) { console.log("Unable to check if lexeme exists on Wikidata, retrying in 10 seconds."); await new Promise((resolve, reject) => setTimeout(resolve, 10000)); return lexemeExists(search, language); } const json = await result.json(); if (!json.search) { console.log("Unable to check if lexeme exists on Wikidata, retrying in 10 seconds."); await new Promise((resolve, reject) => setTimeout(resolve, 10000)); return lexemeExists(search, language); } return json.search.length !== 0; } async function createAdverb(name) { var result = await post.bind(null, "wbeditentity")({ new: "lexeme", summary: "testy bot", bot: true, formatversion: 2, maxlag: 1, data: JSON.stringify({ "type": "lexeme", "claims": [], "language": "Q1806", "lexicalCategory": "Q1001",//"Q380057", "senses": [], "lemmas": { "en": { "language": "en", "value": name } }, "forms": [{ "add": "", "claims": [], "grammaticalFeatures": [], "representations": { "en": { "language": "en", "value": name } } }] }) }).catch(async () => { console.log("Unable to create, retrying in 10 seconds."); await new Promise((resolve, reject) => setTimeout(resolve, 10000)); return await createAdverb(name); }); //console.log(result); if (result.success !== 1) { console.log("Unable to create, retrying in 10 seconds."); await new Promise((resolve, reject) => setTimeout(resolve, 10000)); return await createAdverb(name); } else { return result; } } async function getDictLemmaPage(cmcontinue = "") { const res = await fetch(dictApiUrl + (cmcontinue ? ("&cmcontinue=" + cmcontinue) : "")); let data; try { data = await res.json(); } catch (e) { console.error(await data.text(), "error"); return await getDictLemmaPage(cmcontinue); //throw new Error("bad body"); } if (data.error) { console.log("Unable to search Wiktionary, waiting 15 seconds."); await new Promise((resolve, reject) => setTimeout(resolve, 15000)); return await getDictLemmaPage(cmcontinue); } return data; } async function lemmaCheckLoop(cmcontinue = "", total = 0) { var dictData = await getDictLemmaPage(cmcontinue); let words = []; var promises = []; dictData.query.categorymembers.forEach(async (lexeme, i) => { setTimeout(async () => { let dataCheck = lexemeExists(lexeme.title); promises.push(dataCheck); dataCheck = await dataCheck; total++; if (!dataCheck) { words.push(lexeme); console.log(lexeme.title); } }, i * 34); }); await new Promise((resolve, reject) => {setTimeout(resolve, searchLimit * 34);}); await Promise.all(promises); console.error("checked group, adding adverbs"); for (var i = 0; i < words.length; i++) { console.log("creating", words[i].title); await createAdverb(words[i].title); } if (total > 3) { console.log("total reached"); //return; } if (!dictData.continue || !dictData.continue.cmcontinue) { console.error("no continue"); return; } return lemmaCheckLoop(dictData.continue.cmcontinue, total); } //createAdverb(Math.random().toString(36).split(".")[1]) async function main() { lemmaCheckLoop(); //console.log(await lexemeExists("& al.")) } main();