diff --git a/.ci/eslint-plugin-zotero-translator/lib/rules/test-cases.js b/.ci/eslint-plugin-zotero-translator/lib/rules/test-cases.js index eebcab70816..5e9a67169e5 100644 --- a/.ci/eslint-plugin-zotero-translator/lib/rules/test-cases.js +++ b/.ci/eslint-plugin-zotero-translator/lib/rules/test-cases.js @@ -79,9 +79,16 @@ module.exports = { } else if (testCase.type === 'search') { // console.log(JSON.stringify(testCase.input)) - const expected = ['DOI', 'ISBN', 'PMID', 'identifiers', 'contextObject', 'adsBibcode']; - if (!Object.keys(testCase.input).every(key => expected.includes(key))) { - let invalidKey = Object.keys(testCase.input).find(key => !expected.includes(key)); + const expected = ['DOI', 'ISBN', 'PMID', 'identifiers', 'contextObject', 'adsBibcode', 'ericNumber']; + let keys; + if (Array.isArray(testCase.input)) { + keys = testCase.input.flatMap(Object.keys); + } + else { + keys = Object.keys(testCase.input); + } + if (!keys.every(key => expected.includes(key))) { + let invalidKey = keys.find(key => !expected.includes(key)); context.report({ message: `${prefix} of type "${testCase.type}" has invalid search term '${invalidKey}' - expected one of ${expected.join(', ')}`, loc, diff --git a/.ci/eslint-plugin-zotero-translator/lib/translators.js b/.ci/eslint-plugin-zotero-translator/lib/translators.js index 8bc09362212..8a0bbfef6c9 100755 --- a/.ci/eslint-plugin-zotero-translator/lib/translators.js +++ b/.ci/eslint-plugin-zotero-translator/lib/translators.js @@ -5,7 +5,15 @@ const path = require('path'); const findRoot = require('find-root'); const childProcess = require('child_process'); -const repo = path.resolve(findRoot(__dirname, dir => fs.existsSync(path.resolve(dir, '.git')))); +let repo; +try { + repo = path.resolve(findRoot(__dirname, dir => fs.existsSync(path.resolve(dir, '.git')))); +} +catch (e) { + console.error('ERROR: Translators can only be linted inside a clone of the zotero/translators repo (not a ZIP downloaded from GitHub)'); + console.error(' git clone https://github.com/zotero/translators.git'); + process.exit(1); +} const metaDataRules = [ 'zotero-translator/header-valid-json', diff --git a/.ci/updateTypes.mjs b/.ci/updateTypes.mjs new file mode 100755 index 00000000000..75c074502f2 --- /dev/null +++ b/.ci/updateTypes.mjs @@ -0,0 +1,75 @@ +#!/usr/bin/env node + +import { readFile, writeFile } from 'fs/promises'; + +const INDEX_D_TS_URL = new URL('../index.d.ts', import.meta.url); +const SCHEMA_JSON_URL = new URL('../../zotero-client/resource/schema/global/schema.json', import.meta.url); + +const BEGIN_MARKER = '\t/* *** BEGIN GENERATED TYPES *** */'; +const END_MARKER = '\t/* *** END GENERATED TYPES *** */'; + +async function updateIndexDTS() { + let indexDTS = await readFile(INDEX_D_TS_URL, { encoding: 'utf8' }); + let schema = JSON.parse(await readFile(SCHEMA_JSON_URL)); + + let typeItemTypes = '\ttype ItemTypes = {'; + let itemTypeTypes = ''; + let creatorTypes = new Set(); + + for (let typeSchema of schema.itemTypes) { + let itemType = typeSchema.itemType; + if (['annotation', 'attachment', 'note'].includes(itemType)) { + continue; + } + + let itemTypeUppercase = itemType[0].toUpperCase() + itemType.substring(1) + 'Item'; + if (itemTypeUppercase == 'TvBroadcastItem') { + itemTypeUppercase = 'TVBroadcastItem'; + } + + typeItemTypes += `\n\t\t"${itemType}": ${itemTypeUppercase},`; + itemTypeTypes += `\n\n\ttype ${itemTypeUppercase} = {`; + itemTypeTypes += `\n\t\titemType: "${itemType}";`; + for (let { field } of typeSchema.fields) { + itemTypeTypes += `\n\t\t${field}?: string;` + } + + let creatorTypesJoined = typeSchema.creatorTypes.map(typeSchema => '"' + typeSchema.creatorType + '"').join(' | '); + itemTypeTypes += `\n\n\t\tcreators: Creator<${creatorTypesJoined}>[];`; + itemTypeTypes += '\n\t\tattachments: Attachment[];'; + itemTypeTypes += '\n\t\ttags: Tag[];'; + itemTypeTypes += '\n\t\tnotes: Note[];'; + itemTypeTypes += '\n\t\tseeAlso: string[];'; + itemTypeTypes += '\n\t\tcomplete(): void;'; + itemTypeTypes += '\n\n\t\t[key: string]: string;'; + itemTypeTypes += '\n\t};'; + + for (let { creatorType } of typeSchema.creatorTypes) { + creatorTypes.add(creatorType); + } + } + typeItemTypes += '\n\t};' + + let typeCreatorType = '\n\ttype CreatorType ='; + for (let creatorType of Array.from(creatorTypes).sort()) { + typeCreatorType += `\n\t\t| "${creatorType}"`; + } + typeCreatorType += ';'; + + let beginIdx = indexDTS.indexOf(BEGIN_MARKER); + let endIdx = indexDTS.indexOf(END_MARKER); + if (beginIdx == -1 || endIdx == -1) { + throw new Error('Could not find generated types section in index.d.ts'); + } + + indexDTS = indexDTS.substring(0, beginIdx) + BEGIN_MARKER + '\n' + + typeItemTypes + + itemTypeTypes + + '\n' + typeCreatorType + + '\n' + + indexDTS.substring(endIdx); + + await writeFile(INDEX_D_TS_URL, indexDTS); +} + +updateIndexDTS(); diff --git a/.eslintrc b/.eslintrc index 3ad2bebfd66..ee878c44589 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,14 +1,11 @@ { "env": { "browser": true, - "es2017": true + "es2018": true }, "extends": [ "@zotero" ], - "parserOptions": { - "ecmaVersion": 2018 - }, "globals": { "Zotero": "readonly", "Z": "readonly", diff --git a/ACS Publications.js b/ACS Publications.js index 3aba4ca1c0e..a6782e72481 100644 --- a/ACS Publications.js +++ b/ACS Publications.js @@ -2,76 +2,81 @@ "translatorID": "938ebe32-2b2e-4349-a5b3-b3a05d3de627", "label": "ACS Publications", "creator": "Sean Takats, Michael Berkowitz, Santawort, and Aurimas Vinckevicius", - "target": "^https?://pubs\\.acs\\.org/(toc/|journal/|topic/|isbn/\\d|doi/(full/|abs/)?10\\.|action/doSearch\\?)", + "target": "^https?://pubs\\.acs\\.org/(toc/|journal/|topic/|isbn/\\d|doi/(full/|abs/|epdf/|book/)?10\\.|action/(doSearch\\?|showCitFormats\\?.*doi))", "minVersion": "4.0.5", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2019-10-07 09:04:25" + "lastUpdated": "2023-09-16 00:07:05" } +/* + ***** BEGIN LICENSE BLOCK ***** -function getSearchResults(doc, checkOnly, itemOpts) { + Copyright © 2008 Sean Takats, Michael Berkowitz, Santawort, Aurimas + Vinckevicius, Philipp Zumstein, and other contributors. + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +function getSearchResults(doc, checkOnly) { var items = {}, found = false; var rows = doc.querySelectorAll('.issue-item_title a, .teaser_title a'); for (let i = 0; i < rows.length; i++) { var href = rows[i].href; var title = ZU.trimInternal(rows[i].textContent); + if (!href || !title) continue; var doi = getDoi(href); - if (!href || !title || !doi) continue; + if (!doi) continue; if (checkOnly) return true; found = true; - items[doi] = title; - - // Not sure if this is still working on the new websites... - itemOpts[doi] = {}; - - /* - //check if article contains supporting info, - //so we don't have to waste an HTTP request later if it doesn't - var articleBox = titles[i].parentNode.parentNode; - if (!articleBox.classList.contains('articleBox')) { - // e.g. Most Recently Published under Subject Search - continue; - } - - if (ZU.xpath(articleBox, './/a[text()="Supporting Info"]').length) { - itemOpts[doi].hasSupp = true; - } - */ + items[href] = title; } return found ? items : false; } +// Return the DOI indicated by the URL, or null when no DOI is found +// The input should be a properly encoded URL function getDoi(url) { - var m = url.match(/https?:\/\/[^/]*\/doi\/(?:abs\/|full\/)?(10\.[^?#]+)/); - if (m) { - var doi = m[1]; - if (doi.includes("prevSearch")) { - doi = doi.substring(0, doi.indexOf("?")); - } - return decodeURIComponent(doi); + let urlObj = new URL(url); + let doi = decodeURIComponent(urlObj.pathname).match(/^\/doi\/(?:.+\/)?(10\.\d{4,}\/.+)$/); + if (doi) { + doi = doi[1]; } - return false; + else { + doi = urlObj.searchParams.get("doi"); + } + return doi; } /** *************************** * BEGIN: Supplementary data * *****************************/ -// Get supplementary file names either from the Supporting Info page or the tooltip -function getSuppFiles(div) { - var fileNames = ZU.xpath(div, './/li//li'); - var attach = []; - for (var i = 0, n = fileNames.length; i < n; i++) { - attach.push(fileNames[i].textContent.trim().replace(/\s[\s\S]+/, '')); - } - return attach; -} var suppTypeMap = { + txt: 'text/plain', + csv: 'text/csv', + bz2: 'application/x-bzip2', + gz: 'application/gzip', + zip: 'application/zip', pdf: 'application/pdf', doc: 'application/msword', docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', @@ -79,28 +84,33 @@ var suppTypeMap = { xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' }; -function getSuppMimeType(fileName) { - var ext = fileName.substr(fileName.lastIndexOf('.') + 1); - var mimeType = suppTypeMap[ext]; - return mimeType ? mimeType : undefined; -} - -function attachSupp(item, doi, opts) { - if (!opts.attach) return; - if (!item.attachments) item.attachments = []; - var attachment; - for (var i = 0, n = opts.attach.length; i < n; i++) { - attachment = { - title: opts.attach[i] - }; - attachment.url = '/doi/suppl/' + doi + '/suppl_file/' + attachment.title; - attachment.mimeType = getSuppMimeType(attachment.title); - if (opts.attachAsLink || !attachment.mimeType) { // don't download unknown file types - attachment.snapshot = false; - } - - item.attachments.push(attachment); +function getSupplements(doc, supplementAsLink = false) { + let supplements = []; + // Note that the lists of supplements are duplicated in the main + // content side and right-side panel (if any). We want to confine it to + // one (or the only) side in order to avoid having to deduplicate. + let supplementLinks = doc.querySelectorAll(".article_content-left .suppl-anchor"); + for (let i = 0; i < supplementLinks.length; i++) { + let elem = supplementLinks[i]; + let url = elem.href; + if (!url) continue; + let pathComponents = url.replace(/[?#].+$/, "").split("."); + // possible location of file extension (following the last dot) + let ext = pathComponents[pathComponents.length - 1].toLowerCase(); + let mimeType = suppTypeMap[ext]; + // Only save file when MIME type is known *and* when we aren't + // specifically told otherwise + let snapshot = Boolean(!supplementAsLink && mimeType); + // The "title" (text describing what the supplement file is for) can be + // substantially long, while the filename is redundant (and it doesn't + // inform the user that the file is meant to be a supplement). We + // simply number them in the order they appear. + let title = `Supplement ${i + 1}`; + let attachment = { title, url, snapshot }; + if (mimeType) attachment.mimeType = mimeType; + supplements.push(attachment); } + return supplements; } /** ************************* @@ -111,7 +121,28 @@ function detectWeb(doc, url) { if (getSearchResults(doc, true)) { return "multiple"; } + let urlObj = new URL(url); + // standalone "download citation" page + if (urlObj.pathname === "/action/showCitFormats" + && urlObj.searchParams.get("doi")) { + // May be inaccurate, but better than not detecting + return "journalArticle"; + } + // epdf viewer web app + if (urlObj.pathname.startsWith("/doi/epdf/")) { + // TODO: check if "epdf" viewer is always for journal articles + return "journalArticle"; + } + // books such as https://pubs.acs.org/doi/book/10.1021/acsguide + if (urlObj.pathname.startsWith("/doi/book/")) { + return "book"; + } + if (doc.querySelector("#returnToBook")) { + // Some of them may be conference articles, but the RIS doesn't say so + return "bookSection"; + } else if (getDoi(url)) { + // TODO: check if this block still works var type = doc.getElementsByClassName("content-navigation__contentType"); if (type.length && type[0].textContent.includes("Chapter")) { return "bookSection"; @@ -123,160 +154,117 @@ function detectWeb(doc, url) { return false; } -function doWeb(doc, url) { - var opts = {}; +function cleanNumberField(item, field) { + if (Object.hasOwn(item, field)) { + let n = parseInt(item[field]); + if (n <= 0 || isNaN(n)) { + delete item[field]; + } + } +} + +// In most cases the URL contains the DOI which is sufficient for obtaining the +// RIS, so there's no need to download the document if it's not already there. +// But when supplements as attachments are desired, we need the actual document +// for the supplement links. Our convention here is to pass falsy as the "doc" +// argument when supplements are not requested, and the actual doc (maybe +// fetched by us) when we want the supplements. + +async function doWeb(doc, url) { + let attachSupplement = false; + let supplementAsLink = false; // reduce some overhead by fetching these only once if (Z.getHiddenPref) { - opts.attachSupp = Z.getHiddenPref("attachSupplementary"); - opts.attachAsLink = Z.getHiddenPref("supplementaryAsLink"); + attachSupplement = Z.getHiddenPref("attachSupplementary"); + supplementAsLink = Z.getHiddenPref("supplementaryAsLink"); } - var itemOpts = {}; if (detectWeb(doc, url) == "multiple") { // search - Zotero.selectItems(getSearchResults(doc, false, itemOpts), function (items) { - if (!items) { - return; - } - - var dois = []; - for (let i in items) { - itemOpts[i].pdf = '/doi/pdf/' + i; - dois.push({ doi: i, opts: itemOpts[i] }); - } - - scrape(dois, opts); - }); - } - else { // single article - var doi = getDoi(url); - Zotero.debug("DOI= " + doi); - // we can determine file names from the tooltip, which saves us an HTTP request - var suppTip = doc.getElementById('suppTipDiv'); - if (opts.attachSupp && suppTip) { - try { - opts.attach = getSuppFiles(suppTip, opts); - } - catch (e) { - Z.debug("Error getting supplementary files."); - Z.debug(e); - } + let items = await Z.selectItems(getSearchResults(doc)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape( + attachSupplement && await requestDocument(url), + url, + supplementAsLink + ); + await delay(500); } - - // if we couldn't find this on the individual item page, - // then it doesn't have supp info anyway. This way we know not to check later - if (!opts.attach) opts.attach = []; - - itemOpts.pdf = ZU.xpathText(doc, '(//a[i[contains(@class, "icon-file-pdf-o")]]/@href)[1]') || '/doi/pdf/' + doi; - - scrape([{ doi: doi, opts: itemOpts }], opts); + } + else { + // single article + await scrape(attachSupplement && doc, url, supplementAsLink); } } -function scrape(items, opts) { - for (var i = 0, n = items.length; i < n; i++) { - processCallback(items[i], opts); - } +function delay(milliseconds) { + return new Promise(resolve => setTimeout(resolve, milliseconds)); } -function processCallback(fetchItem, opts) { - var baseurl = "/action/downloadCitation"; - var doi = fetchItem.doi; - var post = "https//pubs.acs.org/action/downloadCitation?direct=true&doi=" + encodeURIComponent(fetchItem.doi) + "&format=ris&include=abs&submit=Download+Citation"; - ZU.doPost(baseurl, post, function (text) { - // Fix the RIS doi mapping - text = text.replace("\nN1 - doi:", "\nDO - "); - // Fix the wrong mapping for journal abbreviations - text = text.replace("\nJO -", "\nJ2 -"); - // Use publication date when available - if (text.includes("\nDA -")) { - text = text.replace(/\nY1 {2}- [^\n]*/, "") - .replace("\nDA -", "\nY1 -"); - } - // Zotero.debug("ris= "+ text); - var translator = Zotero.loadTranslator("import"); - translator.setTranslator("32d59d2d-b65a-4da4-b0a3-bdd3cfb979e7"); - translator.setString(text); - translator.setHandler("itemDone", function (obj, item) { - if (item.date) { - item.date = ZU.strToISO(item.date); - } - item.attachments = []; +async function scrape(doc, url, supplementAsLink) { + let doi = getDoi(url); + + if (doc && /\/action\/showCitFormats\?|\/doi\/epdf\//.test(url)) { + // standalone "export citation" page or "epdf viewer", *and* + // supplements are desired; we need to fetch the actual article page + // and scrape that + url = `https://pubs.acs.org/doi/${doi}`; + doc = await requestDocument(url); + } + + let risURL = new URL("/action/downloadCitation?include=abs&format=ris&direct=true", url); + risURL.searchParams.set("doi", doi); + risURL.searchParams.set("downloadFileName", doi.replace(/^10\.\d{4,}\//, "")); + let risText = await requestText(risURL.href, { headers: { Referer: url } }); + // Delete redundant DOI info + risText = risText.replace(/\nN1 {2}- doi:[^\n]+/, ""); + // Fix noise in DO field + risText = risText.replace("\nDO - doi:", "\nDO - "); + // Fix the wrong mapping for journal abbreviations + risText = risText.replace("\nJO -", "\nJ2 -"); + // Use publication date when available + if (risText.includes("\nDA -")) { + risText = risText.replace(/\nY1 {2}- [^\n]*/, "") + .replace("\nDA -", "\nY1 -"); + } - // standard pdf and snapshot - if (fetchItem.opts.pdf) { + let translator = Zotero.loadTranslator("import"); + // RIS + translator.setTranslator("32d59d2d-b65a-4da4-b0a3-bdd3cfb979e7"); + translator.setString(risText); + translator.setHandler("itemDone", function (obj, item) { + if (item.date) { + item.date = ZU.strToISO(item.date); + } + item.attachments = []; + if (/\/doi\/book\//.test(url)) { + // books as standalone items don't have full pdfs (TODO: verify) + if (doc) { item.attachments.push({ - title: "Full Text PDF", - url: fetchItem.opts.pdf, - mimeType: "application/pdf" + title: "Snapshot", + url: url, + mimeType: "text/html" }); } + } + else { + // standard pdf item.attachments.push({ - title: "ACS Full Text Snapshot", - url: '/doi/full/' + doi, - mimeType: "text/html" + title: "Full Text PDF", + url: `/doi/pdf/${doi}`, + mimeType: "application/pdf" }); - - // supplementary data - try { - if (opts.attachSupp && opts.attach) { - // came from individual item page - attachSupp(item, doi, opts); - } - else if (opts.attachSupp && fetchItem.opts.hasSupp) { - // was a search result and has supp info - var suppUrl = '/doi/suppl/' + doi; - - if (opts.attachAsLink) { - // if we're only attaching links, it's not worth linking to each doc - item.attachments.push({ - title: "Supporting Information", - url: suppUrl, - mimeType: 'text/html', - snapshot: false - }); - } - else { - ZU.processDocuments(suppUrl, function (suppDoc) { - try { - var div = suppDoc.getElementById('supInfoBox'); - if (div) { - var files = getSuppFiles(div); - attachSupp(item, doi, { - attach: files, - attachAsLink: opts.attachAsLink - }); - } - else { - Z.debug("Div not found"); - item.attachments.push({ - title: "Supporting Information", - url: suppUrl, - mimeType: 'text/html', - snapshot: false - }); - } - } - catch (e) { - Z.debug("Error attaching supplementary files."); - Z.debug(e); - } - item.complete(); - }, null, function () { - item.complete(); - }); - return; // don't call item.complete() yet - } - } - } - catch (e) { - Z.debug("Error attaching supplementary files."); - Z.debug(e); - } - - item.complete(); - }); - translator.translate(); + } + // supplements + if (doc) { + item.attachments.push(...getSupplements(doc, supplementAsLink)); + } + // Cleanup fields that may contain invalid numeric values + cleanNumberField(item, "numberOfVolumes"); + cleanNumberField(item, "numPages"); + item.complete(); }); + await translator.translate(); } /** BEGIN TEST CASES **/ @@ -320,10 +308,6 @@ var testCases = [ { "title": "Full Text PDF", "mimeType": "application/pdf" - }, - { - "title": "ACS Full Text Snapshot", - "mimeType": "text/html" } ], "tags": [], @@ -362,7 +346,6 @@ var testCases = [ "bookTitle": "Aquatic Redox Chemistry", "extra": "DOI: 10.1021/bk-2011-1071.ch005", "libraryCatalog": "ACS Publications", - "numberOfVolumes": "0", "pages": "85-111", "publisher": "American Chemical Society", "series": "ACS Symposium Series", @@ -374,10 +357,6 @@ var testCases = [ { "title": "Full Text PDF", "mimeType": "application/pdf" - }, - { - "title": "ACS Full Text Snapshot", - "mimeType": "text/html" } ], "tags": [], @@ -435,10 +414,6 @@ var testCases = [ { "title": "Full Text PDF", "mimeType": "application/pdf" - }, - { - "title": "ACS Full Text Snapshot", - "mimeType": "text/html" } ], "tags": [], @@ -449,18 +424,210 @@ var testCases = [ }, { "type": "web", - "url": "https://pubs.acs.org/isbn/9780841239999", + "url": "https://pubs.acs.org/journal/acbcct", "items": "multiple" }, { "type": "web", - "url": "https://pubs.acs.org/journal/acbcct", + "url": "https://pubs.acs.org/action/doSearch?text1=zotero&field1=AllField", "items": "multiple" }, { "type": "web", - "url": "https://pubs.acs.org/action/doSearch?text1=zotero&field1=AllField", - "items": "multiple" + "url": "https://pubs.acs.org/doi/book/10.1021/acsguide", + "items": [ + { + "itemType": "book", + "title": "The ACS Guide to Scholarly Communication", + "creators": [ + { + "lastName": "Banik", + "firstName": "Gregory M.", + "creatorType": "author" + }, + { + "lastName": "Baysinger", + "firstName": "Grace", + "creatorType": "author" + }, + { + "lastName": "Kamat", + "firstName": "Prashant V.", + "creatorType": "author" + }, + { + "lastName": "Pienta", + "firstName": "Norbert", + "creatorType": "author" + } + ], + "date": "2019-10-02", + "ISBN": "9780841235861", + "extra": "DOI: 10.1021/acsguide", + "libraryCatalog": "ACS Publications", + "publisher": "American Chemical Society", + "series": "ACS Guide to Scholarly Communication", + "url": "https://doi.org/10.1021/acsguide", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://pubs.acs.org/action/showCitFormats?doi=10.1021%2Facscentsci.3c00243", + "items": [ + { + "itemType": "journalArticle", + "title": "Generic Platform for the Multiplexed Targeted Electrochemical Detection of Osteoporosis-Associated Single Nucleotide Polymorphisms Using Recombinase Polymerase Solid-Phase Primer Elongation and Ferrocene-Modified Nucleoside Triphosphates", + "creators": [ + { + "lastName": "Ortiz", + "firstName": "Mayreli", + "creatorType": "author" + }, + { + "lastName": "Jauset-Rubio", + "firstName": "Miriam", + "creatorType": "author" + }, + { + "lastName": "Trummer", + "firstName": "Olivia", + "creatorType": "author" + }, + { + "lastName": "Foessl", + "firstName": "Ines", + "creatorType": "author" + }, + { + "lastName": "Kodr", + "firstName": "David", + "creatorType": "author" + }, + { + "lastName": "Acero", + "firstName": "Josep Lluís", + "creatorType": "author" + }, + { + "lastName": "Botero", + "firstName": "Mary Luz", + "creatorType": "author" + }, + { + "lastName": "Biggs", + "firstName": "Phil", + "creatorType": "author" + }, + { + "lastName": "Lenartowicz", + "firstName": "Daniel", + "creatorType": "author" + }, + { + "lastName": "Trajanoska", + "firstName": "Katerina", + "creatorType": "author" + }, + { + "lastName": "Rivadeneira", + "firstName": "Fernando", + "creatorType": "author" + }, + { + "lastName": "Hocek", + "firstName": "Michal", + "creatorType": "author" + }, + { + "lastName": "Obermayer-Pietsch", + "firstName": "Barbara", + "creatorType": "author" + }, + { + "lastName": "O’Sullivan", + "firstName": "Ciara K.", + "creatorType": "author" + } + ], + "date": "2023-08-23", + "DOI": "10.1021/acscentsci.3c00243", + "ISSN": "2374-7943", + "abstractNote": "Osteoporosis is a multifactorial disease influenced by genetic and environmental factors, which contributes to an increased risk of bone fracture, but early diagnosis of this disease cannot be achieved using current techniques. We describe a generic platform for the targeted electrochemical genotyping of SNPs identified by genome-wide association studies to be associated with a genetic predisposition to osteoporosis. The platform exploits isothermal solid-phase primer elongation with ferrocene-labeled nucleoside triphosphates. Thiolated reverse primers designed for each SNP were immobilized on individual gold electrodes of an array. These primers are designed to hybridize to the SNP site at their 3′OH terminal, and primer elongation occurs only where there is 100% complementarity, facilitating the identification and heterozygosity of each SNP under interrogation. The platform was applied to real blood samples, which were thermally lysed and directly used without the need for DNA extraction or purification. The results were validated using Taqman SNP genotyping assays and Sanger sequencing. The assay is complete in just 15 min with a total cost of 0.3€ per electrode. The platform is completely generic and has immense potential for deployment at the point of need in an automated device for targeted SNP genotyping with the only required end-user intervention being sample addition.", + "issue": "8", + "journalAbbreviation": "ACS Cent. Sci.", + "libraryCatalog": "ACS Publications", + "pages": "1591-1602", + "publicationTitle": "ACS Central Science", + "url": "https://doi.org/10.1021/acscentsci.3c00243", + "volume": "9", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://pubs.acs.org/doi/epdf/10.1021/acscentsci.3c00323", + "items": [ + { + "itemType": "journalArticle", + "title": "Dynamics of Rayleigh Fission Processes in ∼100 nm Charged Aqueous Nanodrops", + "creators": [ + { + "lastName": "Hanozin", + "firstName": "Emeline", + "creatorType": "author" + }, + { + "lastName": "Harper", + "firstName": "Conner C.", + "creatorType": "author" + }, + { + "lastName": "McPartlan", + "firstName": "Matthew S.", + "creatorType": "author" + }, + { + "lastName": "Williams", + "firstName": "Evan R.", + "creatorType": "author" + } + ], + "date": "2023-08-23", + "DOI": "10.1021/acscentsci.3c00323", + "ISSN": "2374-7943", + "abstractNote": "Fission of micron-size charged droplets has been observed using optical methods, but little is known about fission dynamics and breakup of smaller nanosize droplets that are important in a variety of natural and industrial processes. Here, spontaneous fission of individual aqueous nanodrops formed by electrospray is investigated using charge detection mass spectrometry. Fission processes ranging from formation of just two progeny droplets in 2 ms to production of dozens of progeny droplets over 100+ ms are observed for nanodrops that are charged above the Rayleigh limit. These results indicate that Rayleigh fission is a continuum of processes that produce progeny droplets that vary widely in charge, mass, and number.", + "issue": "8", + "journalAbbreviation": "ACS Cent. Sci.", + "libraryCatalog": "ACS Publications", + "pages": "1611-1622", + "publicationTitle": "ACS Central Science", + "url": "https://doi.org/10.1021/acscentsci.3c00323", + "volume": "9", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/AMS MathSciNet.js b/AMS MathSciNet.js index a7a57e5c7c6..03476e79e81 100644 --- a/AMS MathSciNet.js +++ b/AMS MathSciNet.js @@ -9,10 +9,9 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-07-06 14:22:36" + "lastUpdated": "2023-07-14 11:04:37" } - /* ***** BEGIN LICENSE BLOCK ***** @@ -87,8 +86,6 @@ async function scrape(doc, url = doc.location.href) { // Z.debug(bibJSONUrl) let bibJSON = await requestText(bibJSONUrl); // Z.debug(bibJSON) - // the JSON parser doesn't like newlines or backslashes in the bibtex - bibJSON = bibJSON.replace(/\\n/g, "").replace(/\\/g, ""); bibJSON = JSON.parse(bibJSON); let bibTex = bibJSON[0].bib; // Z.debug(bibTex) diff --git a/APA PsycNET.js b/APA PsycNET.js index 994caf73c07..8868d0ae165 100644 --- a/APA PsycNET.js +++ b/APA PsycNET.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-11-02 16:50:48" + "lastUpdated": "2023-08-22 10:02:55" } /* @@ -91,9 +91,9 @@ async function doWeb(doc, url) { if (!items) { return; } - await Promise.all( - Object.keys(items).map(async url => scrape(await requestDocument(url), url)) - ); + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url), url); + } } else { await scrape(doc, url); @@ -132,7 +132,7 @@ async function scrape(doc, url) { var postData = JSON.stringify({ api: "record.exportRISFile", params: { - UIDList: [{UID: uid, ProductCode: productCode}], + UIDList: [{ UID: uid, ProductCode: productCode }], exportType: "zotero" } }); @@ -199,7 +199,7 @@ function processRIS(text, doc) { // try to figure out ids that we can use for fetching RIS async function getIds(doc, url) { - Z.debug('Finding IDs in ' + url) + Z.debug('Finding IDs in ' + url); // try to extract uid from the table var uid = text(doc, '#uid + dd') || text(doc, '#bookUID'); if (uid) { @@ -700,6 +700,11 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://psycnet.apa.org/search/results?id=e6cd5430-40d2-11ee-9aa2-b3e92ca9fef3", + "items": "multiple" } ] /** END TEST CASES **/ diff --git a/Access Engineering.js b/Access Engineering.js index b1c05abce30..3ea7296a56c 100644 --- a/Access Engineering.js +++ b/Access Engineering.js @@ -2,14 +2,14 @@ "translatorID": "d120a8a7-9d45-446e-8c18-ad9ef0a6bf47", "label": "Access Engineering", "creator": "Vinoth K - highwirepress.com", - "target": "^https?://www\\.accessengineeringlibrary\\.com/content/(book|chapter|case-study|video|calculator|tutorial)", + "target": "^https?://www\\.accessengineeringlibrary\\.com/", "minVersion": "3.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-03-07 08:48:13" + "lastUpdated": "2023-09-09 09:42:36" } /* @@ -97,13 +97,16 @@ function scrape(doc, url) { if (edition) item.edition = edition; // Author - let author = ZU.xpath(doc, '//ul[@class="contributor-list"]//li//a'); - if (author.length > 0) { - // Handled using data attribute - for (let i = 0; i < author.length; i++) { - item.creators[i].firstName = author[i].getAttribute('data-firstnames'); - item.creators[i].lastName = author[i].getAttribute('data-surname'); - } + // Some of old pages not having firstname, lastname seperation in markup and ignore if not + let author = doc.querySelectorAll("ul.contributor-list > [data-firstnames]"); + item.creators = []; + for (let i = 0; i < author.length; i++) { + let creatorData = author[i].dataset; + item.creators.push({ + firstName: creatorData.firstnames, + lastName: creatorData.surname, + creatorType: creatorData.authortype + }); } // Abstract @@ -116,12 +119,7 @@ function scrape(doc, url) { translator.getTranslatorObject(function (trans) { // Detect web not get trigger for scape EM translator // - so wll fill those in manually. - if (detectWeb(doc, url)) { - trans.itemType = detectWeb(doc, url); - } - trans.addCustomFields({ - citation_book_title: "bookTitle" - }); + trans.itemType = detectWeb(doc, url); trans.doWeb(doc, url); }); } @@ -130,26 +128,51 @@ function scrape(doc, url) { var testCases = [ { "type": "web", - "url": "https://www.accessengineeringlibrary.com/content/book/9781259860386/", + "url": "https://www.accessengineeringlibrary.com/content/book/9781259860225", "items": [ { "itemType": "book", - "title": "3D Printer Projects for Makerspaces", + "title": "Handbook of Environmental Engineering", "creators": [ { - "firstName": "Lydia Sloan", - "lastName": "Cline", - "creatorType": "author" + "firstName": "Rao Y.", + "lastName": "Surampalli", + "creatorType": "editor" + }, + { + "firstName": "Tian C.", + "lastName": "Zhang", + "creatorType": "editor" + }, + { + "firstName": "Satinder Kaur", + "lastName": "Brar", + "creatorType": "editor" + }, + { + "firstName": "Krishnamoorthy", + "lastName": "Hegde", + "creatorType": "editor" + }, + { + "firstName": "Rama", + "lastName": "Pulicharla", + "creatorType": "editor" + }, + { + "firstName": "Mausam", + "lastName": "Verma", + "creatorType": "editor" } ], - "date": "2017", - "ISBN": "9781259860386", - "abstractNote": "Learn to model and print 3D designs—no experience required!This easy-to-follow guide features twenty 3D printing projects for makers of all skill levels to enjoy. Written in a tutorial, step-by-step manner, 3D Printer Projects for Makerspaces shows how to use Fusion 360, SketchUp, Meshmixer, Remake, and Inkscape to create fun and useful things. Scanning, slicers, silicone molds, settings, and build plate orientation are also covered, as well as post-processing methods that will make your prints really pop!Inside, you9ll learn to model, analyze, and print a:• Phone case• Coin bank• Art stencil• Cookie cutter• Cookie dunker• Personalized key fob• Lens cap holder• Lithophane night-light• Pencil cup with applied sketch• Business card with QR code• Bronze pendant• Soap mold• Hanging lampshade• Scanned Buddha charm• And more!", + "date": "2018", + "ISBN": "9781259860225", + "abstractNote": "A complete guide to environmental regulations and remediation.This practical resource offers thorough coverage of current environmental issues and policies along with step-by-step remediation procedures. With contributions from dozens of industry-recognized experts, Handbook of Environmental Engineering features information on all segments of the market—including water and air quality and hazardous waste—and enables you to ensure compliance with all applicable regulations. You will get details about sensors, monitoring, and toxicity treatment and controls as well as waste management and safe disposal. Real-world examples demonstrate how to apply techniques and achieve compliance, while environmental impact assessments and measurement data enhance the book9s utility.Coverage includes:• Environmental legislation• Environmental impact assessments• Air pollution control and management• Potable water treatment• Wastewater treatment and reuse• Solid waste management• Hazardous waste management• Emerging wastes in the environment• Environmental monitoring and measurements", "edition": "1st Edition", "language": "en", "libraryCatalog": "www.accessengineeringlibrary.com", "publisher": "McGraw-Hill Education", - "url": "https://www.accessengineeringlibrary.com/content/book/9781259860386", + "url": "https://www.accessengineeringlibrary.com/content/book/9781259860225", "attachments": [ { "title": "Full Text PDF", @@ -164,28 +187,61 @@ var testCases = [ }, { "type": "web", - "url": "https://www.accessengineeringlibrary.com/content/book/9781259860386/chapter/chapter12", + "url": "https://www.accessengineeringlibrary.com/content/book/9781259860225/toc-chapter/chapter3/section/section1", "items": [ { "itemType": "bookSection", - "title": "PROJECT 12: Lithophane Night-Light", + "title": "CHAPTER PRELIMINARIES", "creators": [ { - "firstName": "Lydia Sloan", - "lastName": "Cline", + "firstName": "Ashok", + "lastName": "Kumar", "creatorType": "author" + }, + { + "firstName": "Hamid", + "lastName": "Omidvarborna", + "creatorType": "author" + }, + { + "firstName": "Rao Y.", + "lastName": "Surampalli", + "creatorType": "editor" + }, + { + "firstName": "Tian C.", + "lastName": "Zhang", + "creatorType": "editor" + }, + { + "firstName": "Satinder Kaur", + "lastName": "Brar", + "creatorType": "editor" + }, + { + "firstName": "Krishnamoorthy", + "lastName": "Hegde", + "creatorType": "editor" + }, + { + "firstName": "Rama", + "lastName": "Pulicharla", + "creatorType": "editor" + }, + { + "firstName": "Mausam", + "lastName": "Verma", + "creatorType": "editor" } ], - "date": "2017", - "ISBN": "9781259860386", - "abstractNote": "Learn to model and print 3D designs—no experience required!This easy-to-follow guide features twenty 3D printing projects for makers of all skill levels to enjoy. Written in a tutorial, step-by-step manner, 3D Printer Projects for Makerspaces shows how to use Fusion 360, SketchUp, Meshmixer, Remake, and Inkscape to create fun and useful things. Scanning, slicers, silicone molds, settings, and build plate orientation are also covered, as well as post-processing methods that will make your prints really pop!Inside, you'll learn to model, analyze, and print a:• Phone case• Coin bank• Art stencil• Cookie cutter• Cookie dunker• Personalized key fob• Lens cap holder• Lithophane night-light• Pencil cup with applied sketch• Business card with QR code• Bronze pendant• Soap mold• Hanging lampshade• Scanned Buddha charm• And more!", - "bookTitle": "3D Printer Projects for Makerspaces", - "edition": "1st Edition", + "date": "2018", + "ISBN": "9781259860225", + "abstractNote": "A complete guide to environmental regulations and remediation.This practical resource offers thorough coverage of current environmental issues and policies along with step-by-step remediation procedures. With contributions from dozens of industry-recognized experts, Handbook of Environmental Engineering features information on all segments of the market—including water and air quality and hazardous waste—and enables you to ensure compliance with all applicable regulations. You will get details about sensors, monitoring, and toxicity treatment and controls as well as waste management and safe disposal. Real-world examples demonstrate how to apply techniques and achieve compliance, while environmental impact assessments and measurement data enhance the book's utility.Coverage includes:• Environmental legislation• Environmental impact assessments• Air pollution control and management• Potable water treatment• Wastewater treatment and reuse• Solid waste management• Hazardous waste management• Emerging wastes in the environment• Environmental monitoring and measurements", + "bookTitle": "Handbook of Environmental Engineering", "language": "en", "libraryCatalog": "www.accessengineeringlibrary.com", "publisher": "McGraw-Hill Education", - "shortTitle": "PROJECT 12", - "url": "https://www.accessengineeringlibrary.com/content/book/9781259860386/chapter/chapter12", + "url": "https://www.accessengineeringlibrary.com/content/book/9781259860225/toc-chapter/chapter3/section/section1", "attachments": [ { "title": "Snapshot", @@ -200,24 +256,19 @@ var testCases = [ }, { "type": "web", - "url": "https://www.accessengineeringlibrary.com/content/video/V4768153299001", + "url": "https://www.accessengineeringlibrary.com/content/video/V4005352521001", "items": [ { "itemType": "videoRecording", - "title": "10% Infill and a Bridge", - "creators": [ - { - "firstName": "Lydia", - "lastName": "Cline", - "creatorType": "author" - } - ], - "date": "2016", - "abstractNote": "This video shows an item being printed with a 10% infill and includes a bridge.", + "title": "123D Design: Cut Text Through a Plane", + "creators": [], + "date": "2014", + "abstractNote": "This video shows how to cut text through a plane with Combine/Subtract.", "language": "en", "libraryCatalog": "www.accessengineeringlibrary.com", + "shortTitle": "123D Design", "studio": "McGraw-Hill Education", - "url": "https://www.accessengineeringlibrary.com/content/video/V4768153299001", + "url": "https://www.accessengineeringlibrary.com/content/video/V4005352521001", "attachments": [ { "title": "Snapshot", @@ -232,23 +283,23 @@ var testCases = [ }, { "type": "web", - "url": "https://www.accessengineeringlibrary.com/content/calculator/S0018_Analysis_of_AC_and_DC_Circuits_Basic_Calculations", + "url": "https://www.accessengineeringlibrary.com/content/calculator/S0071_Basic_Transformer_Calculations", "items": [ { "itemType": "journalArticle", - "title": "Analysis of A.C. and D.C. Circuits - Basic Calculations", + "title": "Basic Transformer Calculations", "creators": [ { - "firstName": "William", - "lastName": "Prudhomme", + "firstName": "Bhagyalakshmi", + "lastName": "Kerekare", "creatorType": "author" } ], - "date": "2018/12/13/", - "abstractNote": "Software simulation programs are generally used for modeling and designing complex electronic circuits and applications, but frequently only a basic calculation is needed to solve an immediate design problem or to calculate the value of a specific circuit element. This Excel workbook addresses this need by automating the calculation of over 70 basic electronics formulas in direct current (d.c.) and alternating current (a.c.) circuits and applications.", + "date": "2022/06/25/", + "abstractNote": "This Excel workbook contains four worksheets. The first worksheet covers the basic concepts of single phase transformer such as turns ratio, primary current, secondary current, primary voltage, secondary voltage, and transformer ratio calculations. The second worksheet covers the basic concepts of power, efficiency, primary/secondary EMF and transformer rating calculations. The third worksheet covers the basic concepts of three phase transformers, highlighting the star and delta connections. Calculations are done for phase voltage, phase current, line voltage, and line current for star and delta connections. The fourth worksheet covers the basic concepts kVA Ratings, 3-phase primary, and secondary full load current 3-phase voltage calculations.", "language": "en", "libraryCatalog": "www.accessengineeringlibrary.com", - "url": "https://www.accessengineeringlibrary.com/content/calculator/S0018_Analysis_of_AC_and_DC_Circuits_Basic_Calculations", + "url": "https://www.accessengineeringlibrary.com/content/calculator/S0071_Basic_Transformer_Calculations", "attachments": [ { "title": "Snapshot", @@ -295,23 +346,23 @@ var testCases = [ }, { "type": "web", - "url": "https://www.accessengineeringlibrary.com/content/tutorial/T0002_Open_Channel_Flow_Calculations_with_the_Manning_Equation", + "url": "https://www.accessengineeringlibrary.com/content/tutorial/T0004_Partially_Full_Pipe_Flow_Calculations_Using_Excel_Spreadsheets", "items": [ { "itemType": "journalArticle", - "title": "Open Channel Flow Calculations with the Manning Equation using Excel Spreadsheets", + "title": "Partially Full Pipe Flow Calculations Using Excel Spreadsheets", "creators": [ { - "firstName": "Harlan", - "lastName": "H. Bengtson", + "firstName": "Harlan H.", + "lastName": "Bengtson", "creatorType": "author" } ], - "date": "2014-02-01", - "abstractNote": "This tutorial teaches the Manning equation and its use for uniform open channel flow calculations, including the hydraulic radius, Manning roughness coefficient, and normal depth. There are example problems and illustrations show how to use spreadsheets for the calculations.", + "date": "2014/02/01/", + "abstractNote": "This tutorial provides discussion of, and illustration by, examples for use of an Excel spreadsheet for making a variety of calculations for the flow of water in a partially full circular pipe using the Manning Equation. Equations for calculating area, wetted perimeter, and hydraulic radius for partially full pipe flow are included in this tutorial along with a brief review of the Manning Equation and discussion of its use to calculate a) the flow rate in a given pipe (given diameter, slope, & Manning roughness) at a specified depth of flow, b) the required diameter for a specified flow rate at a target percent full in a given pipe, and c) the normal depth (depth of flow) for a specified flow rate in a given pipe. This includes presentation and discussion of the equations for the calculations, example calculations, and screenshots of spreadsheets to facilitate the calculations.", "language": "en", "libraryCatalog": "www.accessengineeringlibrary.com", - "url": "https://www.accessengineeringlibrary.com/content/tutorial/T0002_Open_Channel_Flow_Calculations_with_the_Manning_Equation", + "url": "https://www.accessengineeringlibrary.com/content/tutorial/T0004_Partially_Full_Pipe_Flow_Calculations_Using_Excel_Spreadsheets", "attachments": [ { "title": "Snapshot", diff --git a/Access Science.js b/Access Science.js new file mode 100644 index 00000000000..34be151f37b --- /dev/null +++ b/Access Science.js @@ -0,0 +1,391 @@ +{ + "translatorID": "558330ca-3531-467a-8003-86cd9602cc48", + "label": "Access Science", + "creator": "Vinoth K - highwirepress.com", + "target": "^https?://www\\.accessscience\\.com/", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-10-17 20:19:39" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Vinoth K - highwirepress.com + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { + let title = attr(doc, 'meta[name="citation_title"]', 'content'); + if (title) { + if (doc.querySelector('meta[name="citation_isbn"]')) { + let bookTitle = attr(doc, 'meta[name="citation_book_title"]', 'content'); + if (!bookTitle || title == bookTitle) { + return "book"; + } + else { + return "bookSection"; + } + } + else if (url.includes('content/video/') || url.includes('content/video-biography')) { + return 'videoRecording'; + } + else if (url.includes('content/article/')) { + return "journalArticle"; + } + else if (url.includes('news') || url.includes('briefing')) { + return "magazineArticle"; + } + else { + return "webpage"; + } + } + else if (getSearchResults(doc, true)) { + return "multiple"; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('.search-middle-right a[href]'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +function doWeb(doc, url) { + if (detectWeb(doc, url) == "multiple") { + Zotero.selectItems(getSearchResults(doc, false), function (items) { + if (items) ZU.processDocuments(Object.keys(items), scrape); + }); + } + else { + scrape(doc, url); + } +} + +function scrape(doc, url) { + var translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + translator.setHandler('itemDone', function (obj, item) { + // Author + // Some of video pages having old content which does not contain the + // firstname and lastname. which is binding in a single string in + // metadata tags, So those cases we were split and mapped accordingly + if (item.itemType == 'videoRecording') { + let authorName = attr(doc, 'meta[name="citation_author"]', 'content'); + if (authorName) { + item.creators = []; + if (authorName.includes(',') && authorName.split(',').length > 2) { + authorName = authorName.split(',')[0]; + item.creators.push(ZU.cleanAuthor(authorName, "author", false)); + } + } + } + + let abstractNote = attr(doc, 'meta[name="citation_abstract"]', 'content'); + item.abstractNote = abstractNote && ZU.cleanTags(abstractNote); + + item.complete(); + }); + + translator.getTranslatorObject(function (trans) { + trans.itemType = detectWeb(doc, url); + trans.doWeb(doc, url); + }); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://www.accessscience.com/content/book/9781260452297", + "items": [ + { + "itemType": "book", + "title": "Casarett & Doull's Essentials of Toxicology", + "creators": [ + { + "firstName": "Curtis D.", + "lastName": "Klaassen", + "creatorType": "editor" + }, + { + "firstName": "John B. Watkins", + "lastName": "Iii", + "creatorType": "editor" + } + ], + "date": "2022", + "ISBN": "9781260452297", + "abstractNote": "Doody9s Core Titles for 2021!\n\nFor more than 25 years, Casarett & Doull9s Toxicology: The Basic Science of Poisons has set the standard for providing thorough, academic, and authoritative information in clear and engaging ways. Distilling the major principles and concepts from that renowned text, Casarett & Doull9s Essentials of Toxicology delivers an accessible and highly readable introduction to the science and clinical field of medical toxicology. The book reflects the expertise of more than 60 renowned contributors.\n\nPresented in full-color, this new edition builds on the wide success of previous editions, with extensive updates that make the book more clinically relevant to students and beginners in toxicology, pharmacology, pharmacy, and environmental sciences. Chapter-ending self-assessment Q&As and other features make the learning process more effective and efficient.\n\nCasarett and Doull9s Essentials of Toxicology is organized into seven units:\n\n• General Principles of Toxicology\n\n• Disposition of Toxicants\n\n• Nonorgan-directed Toxicity\n\n• Target Organ Toxicity\n\n• Toxic Agents\n\n• Environmental Toxicology\n\n• Applications of Toxicology\n\nSuccinct, yet comprehensive, the text covers essential principles, toxicokinetics, how toxic effects are passed on to succeeding generations, how each body system responds to poisons, and the specific effects of a wide range of toxic agents—from pesticides to radiation.", + "language": "en", + "libraryCatalog": "www.accessscience.com", + "publisher": "McGraw Hill", + "url": "https://www.accessscience.com/content/book/9781260452297", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessscience.com/content/book/9781260452297/chapter/chapter2", + "items": [ + { + "itemType": "bookSection", + "title": "Principles of Toxicology", + "creators": [ + { + "firstName": "Lauren M.", + "lastName": "Aleksunes", + "creatorType": "author" + }, + { + "firstName": "David L.", + "lastName": "Eaton", + "creatorType": "author" + }, + { + "firstName": "Curtis D.", + "lastName": "Klaassen", + "creatorType": "editor" + }, + { + "firstName": "John B. Watkins", + "lastName": "Iii", + "creatorType": "editor" + } + ], + "date": "2022", + "ISBN": "9781260452297", + "bookTitle": "Casarett & Doull's Essentials of Toxicology", + "language": "en", + "libraryCatalog": "www.accessscience.com", + "publisher": "McGraw Hill", + "url": "https://www.accessscience.com/content/book/9781260452297/chapter/chapter2", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessscience.com/content/video/V2537194263001", + "items": [ + { + "itemType": "videoRecording", + "title": "Supplementary Problem 10.12", + "creators": [ + { + "firstName": "Rebecca B.", + "lastName": "DeVasher", + "creatorType": "author" + } + ], + "date": "2013", + "abstractNote": "This video details a problem involving unit cells and the calculation of the mass of a cell, length of a cell and radius of an atom in the unit cell based on the density of a solid.", + "language": "en", + "libraryCatalog": "www.accessscience.com", + "studio": "McGraw Hill", + "url": "https://www.accessscience.com/content/video/V2537194263001", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessscience.com/content/video-biography/VB0014", + "items": [ + { + "itemType": "videoRecording", + "title": "Anderson, John R.", + "creators": [], + "date": "2011", + "language": "en", + "libraryCatalog": "www.accessscience.com", + "studio": "McGraw Hill", + "url": "https://www.accessscience.com/content/video-biography/VB0014", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessscience.com/content/article/a694300?implicit-login=true", + "items": [ + { + "itemType": "journalArticle", + "title": "3D printing", + "creators": [ + { + "firstName": "Wenchao", + "lastName": "Zhou", + "creatorType": "author" + } + ], + "date": "2023", + "DOI": "10.1036/1097-8542.694300", + "language": "en", + "libraryCatalog": "www.accessscience.com", + "url": "https://www.accessscience.com/content/article/a694300", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessscience.com/content/news/aSN2301171?implicit-login=true", + "items": [ + { + "itemType": "magazineArticle", + "title": "These chemists cracked the code to long-lasting Roman concrete", + "creators": [ + { + "firstName": "Carolyn", + "lastName": "Gramling", + "creatorType": "author" + } + ], + "date": "2023", + "extra": "DOI: 10.1036/1097-8542.SN0000000", + "language": "en", + "libraryCatalog": "www.accessscience.com", + "url": "https://www.accessscience.com/content/news/aSN2301171", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessscience.com/content/video/an600010", + "items": [ + { + "itemType": "videoRecording", + "title": "Henrietta Leavitt: The Woman Who Measured the Universe", + "creators": [], + "abstractNote": "Born in 1868, Henrietta Leavitt was an astronomer ahead of her time, whose work helped to revolutionize our understanding of the universe. While working at Harvard Observatory, Leavitt began to study stars of fluctuating brightness. This video describes her crucial observation about variable stars, which gave astronomers a new way to measure distances, ultimately leading to such impactful discoveries as the expansion of the universe.\n\nCredit: ESA Hubble Videos; Hubblecast 116: Henrietta Leavitt — ahead of her time; Directed by: Mathias Jäger; Visual design and editing: Martin Kornmesser; Written by: Sara Rigby; Narration: Sara Mendes da Costa; Images: ESA/Hubble and NASA, ESO, Hubble Heritage Team (STScI/AURA), Library of Congress Prints and Photographs Division Washington, Harvard College Observatory, Huntington Library, California Institute of Technology, Digitized Sky Survey 2, M. Kornmesser, R. Gendler, Arnold Reinhold, Davide De Martin; Videos: NASA, ESA, M. Kornmesser, Luis Calcada; Music: Johan B. Monell; Web and technical support: Mathias André and Raquel Yumi Shida; Executive producer: Lars Lindberg Christensen", + "language": "en", + "libraryCatalog": "www.accessscience.com", + "shortTitle": "Henrietta Leavitt", + "studio": "McGraw Hill", + "url": "https://www.accessscience.com/content/video/an600010", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessscience.com/content/biography/m0073908", + "items": [ + { + "itemType": "webpage", + "title": "Abbe, Cleveland (1838–1916)", + "creators": [], + "language": "en", + "url": "https://www.accessscience.com/content/biography/m0073908", + "websiteTitle": "McGraw Hill's AccessScience", + "websiteType": "text", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.accessscience.com/search?query=&items_per_page=10", + "items": "multiple" + } +] +/** END TEST CASES **/ diff --git a/Airiti.js b/Airiti.js index e6797daa58e..a20b1612280 100644 --- a/Airiti.js +++ b/Airiti.js @@ -7,9 +7,9 @@ "maxVersion": "", "priority": 110, "inRepository": true, - "translatorType": 12, + "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-02-17 09:30:16" + "lastUpdated": "2023-08-04 05:04:16" } function detectWeb(doc, url) { @@ -112,419 +112,312 @@ function buildQuery(docIDs) { url += '&DocIDs[' + i + ']=' + encodeURIComponent(docIDs[i]); } return url; -} - -// TODO: Re-enable search -// e.g. 10.6220/joq.2012.19(1).01 -function detectSearch(items) { - if (!items) return false; - - if (typeof items == 'string' || !items.length) items = [items]; - - for (var i=0, n=items.length; i a[href*="/Articles/"]'); + var rows = doc.querySelectorAll('td > a[href^="/Articles/"], th > a[href^="/Articles/"]'); for (let row of rows) { - // TODO: check and maybe adjust let href = row.href; - // TODO: check and maybe adjust let title = ZU.trimInternal(row.textContent); if (!href || !title) continue; if (checkOnly) return true; @@ -66,45 +68,47 @@ function getSearchResults(doc, checkOnly) { return found ? items : false; } -function doWeb(doc, url) { +async function doWeb(doc, url) { if (detectWeb(doc, url) == "multiple") { - Zotero.selectItems(getSearchResults(doc, false), function (items) { - if (items) ZU.processDocuments(Object.keys(items), scrape); - }); + let items = await Z.selectItems(getSearchResults(doc)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(null, url); + } } else { - scrape(doc, url); + await scrape(doc, url); } } -function scrape(doc, url) { +async function scrape(doc, url) { let item = new Zotero.Item('magazineArticle'); let [, MID, IID, AID] = url.match(urlRe); - ZU.doGet(`${apiBase}/Search/IssueHInfo?MID=${MID}&IID=${IID}`, function (respText) { - let issue = JSON.parse(respText); - ZU.doGet(`${apiBase}/Search/ArticleHInfo?AID=${AID}`, function (respText) { - let article = JSON.parse(respText); - - item.title = article.articleTitle.replace(' : ', ": "); - item.pages = article.pageNo; - item.creators.push(ZU.cleanAuthor(article.articleAuthor, 'author')); - - item.publicationTitle = issue.magazineArabicName; - item.place = issue.countryName; - item.issue = issue.issuenumber || issue.issueName; - item.date = ZU.strToISO(arabicToEnglishDate(issue.newIssueDate)); - - item.url = url; + let issue = await requestJSON(`${apiBase}/Search/IssueHInfo?MID=${MID}&IID=${IID}`); + let article = await requestJSON(`${apiBase}/Search/ArticleHInfo?AID=${AID}`); - item.attachments.push({ - title: 'Snapshot', - document: doc - }); - - item.complete(); - }); - }); + item.title = article.articleTitle.replace(' : ', ": "); + item.pages = article.pageNo; + item.creators.push(ZU.cleanAuthor(article.articleAuthor, 'author')); + + item.publicationTitle = issue.magazineArabicName; + item.place = issue.countryName; + item.issue = issue.issuenumber || issue.issueName; + item.date = ZU.strToISO(arabicToEnglishDate(issue.newIssueDate)); + + item.url = url; + + let attachment = { title: "Snapshot" }; + if (doc) { + attachment.document = doc; + } + else { + attachment.url = url; + } + item.attachments.push(attachment); + + item.complete(); } // just so we get months on non-Arabic locales @@ -226,6 +230,12 @@ var testCases = [ { "type": "web", "url": "https://archive.alsharekh.org/contents/174/19785", + "defer": true, + "items": "multiple" + }, + { + "type": "web", + "url": "https://archive.alsharekh.org/AuthorArticles/124", "items": "multiple" } ] diff --git a/AquaDocs.js b/AquaDocs.js new file mode 100644 index 00000000000..00186949864 --- /dev/null +++ b/AquaDocs.js @@ -0,0 +1,390 @@ +{ + "translatorID": "97b65138-71b7-424f-b305-4a2161e90661", + "label": "AquaDocs", + "creator": "Sebastian Karcher", + "target": "^https?://aquadocs\\.org/(handle|discover|browse)", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-08-24 02:41:29" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Sebastian Karcher + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + + +function detectWeb(doc, url) { + if (url.includes('/handle/') && attr(doc, 'meta[name="DC.type"]', 'content')) { + let type = attr(doc, 'meta[name="DC.type"]', 'content'); + // Z.debug(type); + return getType(type); + } + else if (getSearchResults(doc, true)) { + return 'multiple'; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('.main-content .description-content a'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(url); + } + } + else { + await scrape(url); + } +} + +function getType(string) { + string = string.toLowerCase(); + if (string.includes("book_section") || string.includes("chapter")) { + return "bookSection"; + } + else if (string.includes("book") || string.includes("monograph")) { + return "book"; + } + else if (string.includes("report")) { + return "report"; + } + else if (string.includes("proceedings") || string.includes("conference")) { + return "conferencePaper"; + } + else { + return "journalArticle"; // default -- most of the catalog + } +} + +async function scrape(url) { + let xmlURL = url.replace("/handle/", "/metadata/handle/").replace(/[?#].*$/, "") + "/mets.xml"; + // Z.debug(xmlURL); + let xmlText = await requestText(xmlURL); + // Z.debug(xmlText) + let translator = Zotero.loadTranslator('import'); + translator.setTranslator('2c05e2d1-a533-448f-aa20-e919584864cb'); // DIM + translator.setString(xmlText); + translator.setHandler('itemDone', (_obj, item) => { + for (let attachment of item.attachments) { + if (attachment.url && !attachment.url.startsWith("http")) { + attachment.url = "https://aquadocs.org" + attachment.url; + } + } + item.complete(); + }); + await translator.translate(); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://aquadocs.org/handle/1834/42391", + "items": [ + { + "itemType": "journalArticle", + "title": "Upwelling phenomenon in the marine regions of Southern Central of Vietnam: a review", + "creators": [ + { + "firstName": "Hong Long", + "lastName": "Bui", + "creatorType": "author" + }, + { + "firstName": "Minh Thu", + "lastName": "Phan", + "creatorType": "author" + } + ], + "date": "2022", + "ISSN": "1859-3097", + "abstractNote": "Upwelling is an oceanographic phenomenon that involves the physical process and contributes to changes in chemistry, biology, and natural resources. So, systematically, it is the particular ecosystems of whole marine regions with the upwelling. The strong upwelling waters in South Central Regions of Vietnam have uncertain features of the East Vietnam Sea (Bien Dong) and special characteristics of a coastal upwelling area, recorded in international scientific papers in the twentieth century. Their first signals were discovered in the early 1930s through conceptual ideas. The upwelling phenomenon is officially confirmed by scientific results of marine investigations of the NAGA Expedition (1959–1961). The paper aims to review and discuss the physical from Vietnamese investigation and results since 1990s. The following factors are the most contributing to forming and developing the strong upwelling in Southern Central Waters: (1) Influence scale (Mezo- and micro-scale); (2) Forming causes and developing mechanism of upwelling phenomenon, such as monsoon, morphography, shoreline, and western boundary current system of the East Vietnam Sea; (3) Influence of the water-mass from Mekong River on the upwelling area; (4) Ecological environmental consequences; (5) Impacts of the atmospheric-oceanic interaction processes on the western EVS on upwelling. Additionally, the review has targeted findings of upwelling phenomenon mainly in Vietnamese waters based on remote sensing analysis and reanalysis data series to simulate their forming, mechanizing, fluctuating models and the impacts of upwelling in the EVS on resources and ecosystems. The coupled atmosphere-ocean models resulted the upwelling mechanisms and formation. The long-time series of upwelling phenomenon (Macroscale) were evaluated by remote sensing and reanalyzed data series. It is also providing the supplementing and detailing causes and mechanisms of upwelling formation; impacts and interactions of upwelling on marine physics and hydrodynamics (ocean vortexes, seawater temperature), biochemical (nutrients, plankton organisms), and resources (fish, seafood). Within the framework of strong upwelling waters in the Southern Central Regions (Vietnam), the review has not only mentioned partly clarified scientific results but also indicates the limitations and challenges which were faced and encountered in the forecasters of upwelling phenomena in the future.", + "issue": "2", + "language": "en", + "libraryCatalog": "AquaDocs", + "pages": "103-122", + "publicationTitle": "Vietnam of Journal Marine Science and Technology", + "shortTitle": "Upwelling phenomenon in the marine regions of Southern Central of Vietnam", + "url": "http://hdl.handle.net/1834/42391", + "volume": "22", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "Natural resources" + }, + { + "tag": "Upwelling phenomenon" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://aquadocs.org/handle/1834/20117?show=full", + "items": [ + { + "itemType": "book", + "title": "M/V CONNECTED Coral Reef Restoration Monitoring Report, Monitoring Events 2004-2005. Florida Keys National Marine Sanctuary Monroe County, Florida", + "creators": [ + { + "firstName": "Joe", + "lastName": "Schittone", + "creatorType": "author" + }, + { + "firstName": "Erik C.", + "lastName": "Franklin", + "creatorType": "author" + }, + { + "firstName": "J. Harold", + "lastName": "Hudson", + "creatorType": "author" + }, + { + "firstName": "Jeff", + "lastName": "Anderson", + "creatorType": "author" + } + ], + "date": "2006", + "abstractNote": "This document presents the results of the monitoring of a repaired coral reef injured by the M/V Connected vessel grounding incident of March 27, 2001. This groundingoccurred in Florida state waters within the boundaries of the Florida Keys National Marine Sanctuary (FKNMS). The National Oceanic and Atmospheric Administration (NOAA) and the Board of Trustees of the Internal Improvement Trust Fund of the State of Florida, (“State of Florida” or “state”) are the co-trustees for the natural resourceswithin the FKNMS and, thus, are responsible for mediating the restoration of the damaged marine resources and monitoring the outcome of the restoration actions. Therestoration monitoring program tracks patterns of biological recovery, determines the success of restoration measures, and assesses the resiliency to environmental andanthropogenic disturbances of the site over time.The monitoring program at the Connected site was to have included an assessment of the structural stability of installed restoration modules and biological condition of reattached corals performed on the following schedule: immediately (i.e., baseline), 1, 3, and 6 years after restoration and following a catastrophic event. Restoration of this site was completed on July 20, 2001. Due to unavoidable delays in the settlement of the case, the“baseline” monitoring event for this site occurred in July 2004. The catastrophic monitoring event occurred on August 31, 2004, some 2 ½ weeks after the passage of Hurricane Charley which passed nearby, almost directly over the Dry Tortugas. In September 2005, the year one monitoring event occurred shortly after the passage of Hurricane Katrina, some 70 km to the NW. This report presents the results of all three monitoring events. (PDF contains 37 pages.)", + "language": "en", + "libraryCatalog": "AquaDocs", + "place": "Silver Spring, MD", + "publisher": "NOAA/National Ocean Service/National Marine Sanctuary Program", + "series": "Marine Sanctuaries Conservation Series", + "url": "http://hdl.handle.net/1834/20117", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "Acropora palmata" + }, + { + "tag": "Coral" + }, + { + "tag": "Ecology" + }, + { + "tag": "Environment" + }, + { + "tag": "Florida Keys National Marine Sanctuary" + }, + { + "tag": "Grounding" + }, + { + "tag": "Hurricane Charley" + }, + { + "tag": "Hurricane Katrina" + }, + { + "tag": "Management" + }, + { + "tag": "Monitoring" + }, + { + "tag": "Restoration" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://aquadocs.org/discover", + "items": "multiple" + }, + { + "type": "web", + "url": "https://aquadocs.org/browse?type=subject&value=A.+gueldenstaedtii", + "items": "multiple" + }, + { + "type": "web", + "url": "https://aquadocs.org/handle/1834/30052?show=full", + "items": [ + { + "itemType": "bookSection", + "title": "Ecological Attribute Alteration: Measurement and Evaluation: Activity Assessment Routine: Ecological Systems Component, Ecological Systems Component Handbook", + "creators": [], + "date": "1978-08", + "abstractNote": "This technical paper is intended to provide a more complete treatment of implicit principles and assumptions contained in the user's manual for the ecological systems component of the activity assessment routine. The ecological systems component (ESC) defines a method for evaluating changes in an ecosystem which may result from resource use and consumption. This paper begins by characterizing an ecosystem as an organized collection of attributes mutually dependent on energy exchange. The magnitude matrix with which altered energy flows are scaled is described in Chapter 4. The magnitude of an alteration is assessed somewhat differently for the two categories of attributes: discussion of conventions relevant to this distinction is provided in Chapter 5. However, effects on attributes are variable through time, and additional remarks concerning duration are included in Chapter 6. Finally, possible exceptions to the general guidelines for designating the direction of an effect are introduced in Chapter 7.", + "bookTitle": "Ecological Systems Component Handbook", + "language": "en", + "libraryCatalog": "AquaDocs", + "place": "Austin, TX", + "publisher": "RPC, Inc.", + "series": "Technical Paper", + "shortTitle": "Ecological Attribute Alteration", + "url": "http://hdl.handle.net/1834/30052", + "attachments": [], + "tags": [ + { + "tag": "Ecology" + }, + { + "tag": "Management" + }, + { + "tag": "coastal zone management" + }, + { + "tag": "ecological assessment" + }, + { + "tag": "evaluation" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://aquadocs.org/handle/1834/970?show=full", + "items": [ + { + "itemType": "conferencePaper", + "title": "Report from the WPB on the Data Situation for Billfish", + "creators": [], + "date": "2000", + "conferenceName": "IOTC 3", + "language": "en", + "libraryCatalog": "AquaDocs", + "pages": "102-103", + "proceedingsTitle": "IOTC Proceedings no. 3", + "publisher": "IOTC", + "url": "http://hdl.handle.net/1834/970", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "Billfisheries" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://aquadocs.org/handle/1834/31638?show=full", + "items": [ + { + "itemType": "conferencePaper", + "title": "Introduction", + "creators": [ + { + "firstName": "Caroline M.", + "lastName": "Isaacs", + "creatorType": "author" + } + ], + "date": "1997", + "abstractNote": "The Thirteenth Annual PACLIM Workshop was held at the Asilomar Conference Center on April 14-17, 1996. Attended by about 100 registered participants, the workshop included 27 talks and 26 poster presentations. The talks consisted of a one-day theme session of seven 45-minute talks and two featured evening talks. Throughout the remainder of the meeting were nearly 20 shorter, 20-minute presentations. Poster presenters gave a short 1-2 minute introduction to their posters, which were displayed during the entire meeting.All presenters were invited to expand their abstracts into a manuscript for inclusion in the Proceedings volume, and nearly all presentations are included in manuscript or abstract form. In this Proceedings volume, manuscripts are presented first, and abstracts of talks and then posters follow.", + "conferenceName": "Thirteenth Annual Pacific Climate (PACLIM) Workshop", + "language": "en", + "libraryCatalog": "AquaDocs", + "pages": "1-8", + "url": "http://hdl.handle.net/1834/31638", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "Atmospheric Sciences" + }, + { + "tag": "Earth Sciences" + }, + { + "tag": "Ecology" + }, + { + "tag": "Limnology" + }, + { + "tag": "Oceanography" + }, + { + "tag": "PACLIM" + }, + { + "tag": "hydrology" + } + ], + "notes": [], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/AustLII and NZLII.js b/AustLII and NZLII.js index 49f04ecac13..0ef4dcae009 100644 --- a/AustLII and NZLII.js +++ b/AustLII and NZLII.js @@ -9,14 +9,14 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2018-03-02 09:46:09" + "lastUpdated": "2023-08-03 04:51:32" } /* ***** BEGIN LICENSE BLOCK ***** Copyright © 2018 Justin Warren, Philipp Zumstein - + This file is part of Zotero. Zotero is free software: you can redistribute it and/or modify @@ -35,15 +35,9 @@ ***** END LICENSE BLOCK ***** */ - -// attr()/text() v2 -function attr(docOrElem,selector,attr,index){var elem=index?docOrElem.querySelectorAll(selector).item(index):docOrElem.querySelector(selector);return elem?elem.getAttribute(attr):null;}function text(docOrElem,selector,index){var elem=index?docOrElem.querySelectorAll(selector).item(index):docOrElem.querySelector(selector);return elem?elem.textContent:null;} - - function detectWeb(doc, url) { var classes = attr(doc, 'body', 'class'); - // Z.debug(classes); - + if (classes.includes('case')) { return "case"; } @@ -62,6 +56,7 @@ function detectWeb(doc, url) { if (getSearchResults(doc, true)) { return "multiple"; } + return false; } @@ -69,7 +64,7 @@ function getSearchResults(doc, checkOnly) { var items = {}; var found = false; var rows = doc.querySelectorAll('#page-main ul>li>a'); - for (let i=0; ia>span'); + var full_jurisdiction = text(doc, 'li.ribbon-jurisdiction>a>span'); + var jurisdiction = jurisdictionAbbrev[full_jurisdiction] || full_jurisdiction; if (jurisdiction) { - newItem.extra = "jurisdiction: " + jurisdiction; + newItem.code = jurisdiction; } var citation = text(doc, 'li.ribbon-citation>a>span'); - - + var voliss; + var m; + if (text(doc, '#ribbon')) { if (type == "case") { - var voliss = text(doc, 'head>title'); + voliss = text(doc, 'head>title'); // e.g. C & M [2006] FamCA 212 (20 January 2006) newItem.caseName = voliss.replace(/\s?\[.*$/, ''); newItem.title = newItem.caseName; - - var lastParenthesis = voliss.match(/\(([^\)]*)\)$/); + + var lastParenthesis = voliss.match(/\(([^)]*)\)$/); if (lastParenthesis) { newItem.dateDecided = ZU.strToISO(lastParenthesis[1]); - } else { + } + else { newItem.dateDecided = text(doc, 'li.ribbon-year>a>span'); } - newItem.court = text(doc, 'li.ribbon-database>a>span'); + var full_court = text(doc, 'li.ribbon-database>a>span'); + newItem.court = courtAbbrev[full_court] || full_court; if (citation) { var lastNumber = citation.match(/(\d+)$/); if (lastNumber) { @@ -133,49 +202,54 @@ function scrape(doc, url) { } } if (type == "statute") { - // title - newItem.nameOfAct = citation.trim(); + // All AustLII Act titles end in the year the Act was passed + const actInfo = parseActName(citation); + newItem.nameOfAct = actInfo.actName; + newItem.dateEnacted = actInfo.actYear; // section newItem.section = text(doc, 'li.ribbon-subject>a>span'); if (newItem.section) newItem.section = newItem.section.replace(/^SECT /, ''); } if (type == "journalArticle") { var title = text(doc, 'title'); - var m = title.match(/(.*) --- "([^"]*)"/); + m = title.match(/(.*) --- "([^"]*)"/); if (m) { newItem.title = m[2]; var authors = m[1].split(';'); - for (let i=0; ia>span'); newItem.date = text(doc, 'li.ribbon-year>a>span'); } - } else { - var voliss = text(doc, 'head>title'); + } + else { + voliss = text(doc, 'head>title'); // e.g. C & M [2006] FamCA 212 (20 January 2006) - var m = voliss.match(/^([^[]*)\[(\d+)\](.*)\(([^\)]*)\)$/); + m = voliss.match(/^([^[]*)\[(\d+)\](.*)\(([^)]*)\)$/); if (m) { newItem.title = m[1]; newItem.dateDecided = ZU.strToISO(m[4]); var courtNumber = m[3].trim().split(' '); - if (courtNumber.length>=2) { + if (courtNumber.length >= 2) { newItem.court = courtNumber[0]; newItem.docketNumber = courtNumber[1].replace(/[^\w]*$/, ''); } - } else { + } + else { newItem.title = voliss; } } - + newItem.url = url; newItem.attachments = [{ document: doc, title: "Snapshot", - mimeType:"text/html" + mimeType: "text/html" }]; newItem.complete(); } @@ -184,17 +258,17 @@ function scrape(doc, url) { var testCases = [ { "type": "web", - "url": "http://www7.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/FamCA/2006/212.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/FamCA/2006/212.html", "items": [ { "itemType": "case", "caseName": "C & M", "creators": [], "dateDecided": "2006-01-20", - "court": "Family Court of Australia", + "code": "Cth", + "court": "FamCA", "docketNumber": "212", - "extra": "jurisdiction: Commonwealth", - "url": "http://www7.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/FamCA/2006/212.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/FamCA/2006/212.html", "attachments": [ { "title": "Snapshot", @@ -209,17 +283,17 @@ var testCases = [ }, { "type": "web", - "url": "http://www8.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/FCA/2010/1.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/FCA/2010/1.html", "items": [ { "itemType": "case", "caseName": "Yeo, in the matter of AES Services (Aust) Pty Ltd (ACN 111 306 543) (Administrators Appointed)", "creators": [], "dateDecided": "2010-01-05", - "court": "Federal Court of Australia", + "code": "Cth", + "court": "FCA", "docketNumber": "1", - "extra": "jurisdiction: Commonwealth", - "url": "http://www8.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/FCA/2010/1.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/FCA/2010/1.html", "attachments": [ { "title": "Snapshot", @@ -258,22 +332,22 @@ var testCases = [ }, { "type": "web", - "url": "http://www8.austlii.edu.au/cgi-bin/viewtoc/au/cases/act/ACTSC/2010/", + "url": "http://www.austlii.edu.au/cgi-bin/viewtoc/au/cases/act/ACTSC/2010/", "items": "multiple" }, { "type": "web", - "url": "http://www8.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/AICmr/2017/134.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/AICmr/2017/134.html", "items": [ { "itemType": "case", "caseName": "'NM' and Department of Human Services (Freedom of information)", "creators": [], "dateDecided": "2017-12-08", - "court": "Australian Information Commissioner", + "code": "Cth", + "court": "AICmr", "docketNumber": "134", - "extra": "jurisdiction: Commonwealth", - "url": "http://www8.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/AICmr/2017/134.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/AICmr/2017/134.html", "attachments": [ { "title": "Snapshot", @@ -288,15 +362,39 @@ var testCases = [ }, { "type": "web", - "url": "http://www8.austlii.edu.au/cgi-bin/viewdoc/au/legis/cth/consol_act/foia1982222/s24ab.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/legis/cth/consol_act/foia1982222/s24ab.html", "items": [ { "itemType": "statute", - "nameOfAct": "Freedom of Information Act 1982", + "nameOfAct": "Freedom of Information Act", "creators": [], - "extra": "jurisdiction: Commonwealth", + "dateEnacted": "1982", + "code": "Cth", "section": "24AB", - "url": "http://www8.austlii.edu.au/cgi-bin/viewdoc/au/legis/cth/consol_act/foia1982222/s24ab.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/legis/cth/consol_act/foia1982222/s24ab.html", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "http://www.austlii.edu.au/cgi-bin/viewdb/au/legis/cth/consol_act/foia1982222/", + "items": [ + { + "itemType": "statute", + "nameOfAct": "Freedom of Information Act", + "creators": [], + "dateEnacted": "1982", + "code": "Cth", + "url": "http://www.austlii.edu.au/cgi-bin/viewdb/au/legis/cth/consol_act/foia1982222/", "attachments": [ { "title": "Snapshot", @@ -311,14 +409,15 @@ var testCases = [ }, { "type": "web", - "url": "http://www8.austlii.edu.au/cgi-bin/viewdb/au/legis/cth/consol_act/foia1982222/", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/cth/consol_act/antsasta1999402/", "items": [ { "itemType": "statute", - "nameOfAct": "Freedom of Information Act 1982", + "nameOfAct": "A New Tax System (Goods and Services Tax) Act", "creators": [], - "extra": "jurisdiction: CTH", - "url": "http://www8.austlii.edu.au/cgi-bin/viewdb/au/legis/cth/consol_act/foia1982222/", + "dateEnacted": "1999", + "code": "Cth", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/cth/consol_act/antsasta1999402/", "attachments": [ { "title": "Snapshot", @@ -333,7 +432,30 @@ var testCases = [ }, { "type": "web", - "url": "http://www9.austlii.edu.au/cgi-bin/viewdoc/au/journals/AdminRw//2010/9.html", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/cth/consol_act/caca2010265/", + "items": [ + { + "itemType": "statute", + "nameOfAct": "Competition and Consumer Act", + "creators": [], + "dateEnacted": "2010", + "code": "Cth", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/cth/consol_act/caca2010265/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/journals/AdminRw//2010/9.html", "items": [ { "itemType": "journalArticle", @@ -353,7 +475,7 @@ var testCases = [ "date": "2010", "libraryCatalog": "AustLII and NZLII", "publicationTitle": "Administrative Review Council - Admin Review", - "url": "http://www9.austlii.edu.au/cgi-bin/viewdoc/au/journals/AdminRw//2010/9.html", + "url": "http://www.austlii.edu.au/cgi-bin/viewdoc/au/journals/AdminRw//2010/9.html", "attachments": [ { "title": "Snapshot", @@ -368,12 +490,12 @@ var testCases = [ }, { "type": "web", - "url": "http://www7.austlii.edu.au/cgi-bin/sinosrch.cgi?mask_path=;method=auto;query=adam%20smith;view=relevance&mask_path=au/cases/act/ACTCA", + "url": "http://www.austlii.edu.au/cgi-bin/sinosrch.cgi?mask_path=;method=auto;query=adam%20smith;view=relevance&mask_path=au/cases/act/ACTCA", "items": "multiple" }, { "type": "web", - "url": "http://www8.austlii.edu.au/cgi-bin/sinodisp/au/cases/cth/AICmr/2017/20.html", + "url": "http://www.austlii.edu.au/cgi-bin/sinodisp/au/cases/cth/AICmr/2017/20.html", "items": [ { "itemType": "case", @@ -382,7 +504,76 @@ var testCases = [ "dateDecided": "2017-03-10", "court": "AICmr", "docketNumber": "20", - "url": "http://www8.austlii.edu.au/cgi-bin/sinodisp/au/cases/cth/AICmr/2017/20.html", + "url": "http://www8.austlii.edu.au/cgi-bin/viewdoc/au/cases/cth/AICmr/2017/20.html", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/qld/consol_act/pla1974179/", + "items": [ + { + "itemType": "statute", + "nameOfAct": "Property Law Act", + "creators": [], + "dateEnacted": "1974", + "code": "Qld", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/qld/consol_act/pla1974179/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/vic/consol_act/ca195882/", + "items": [ + { + "itemType": "statute", + "nameOfAct": "Crimes Act", + "creators": [], + "dateEnacted": "1958", + "code": "Vic", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/vic/consol_act/ca195882/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/nsw/consol_act/leara2002451/", + "items": [ + { + "itemType": "statute", + "nameOfAct": "Law Enforcement (Powers and Responsibilities) Act", + "creators": [], + "dateEnacted": "2002", + "code": "NSW", + "url": "https://www.austlii.edu.au/cgi-bin/viewdb/au/legis/nsw/consol_act/leara2002451/", "attachments": [ { "title": "Snapshot", diff --git a/Boston Review.js b/Boston Review.js index eed191b738b..b583cedda3f 100644 --- a/Boston Review.js +++ b/Boston Review.js @@ -3,92 +3,154 @@ "label": "Boston Review", "creator": "Sebastian Karcher", "target": "^https?://(www\\.)?bostonreview\\.net/", - "minVersion": "2.1.9", + "minVersion": "6.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2015-08-09 19:58:48" + "lastUpdated": "2023-08-18 06:17:49" } -/* FW LINE 59:b820c6d */ function flatten(t){var e=new Array;for(var i in t){var r=t[i];r instanceof Array?e=e.concat(flatten(r)):e.push(r)}return e}var FW={_scrapers:new Array};FW._Base=function(){this.callHook=function(t,e,i,r){if("object"==typeof this.hooks){var n=this.hooks[t];"function"==typeof n&&n(e,i,r)}},this.evaluateThing=function(t,e,i){var r=typeof t;if("object"===r){if(t instanceof Array){var n=this.evaluateThing,a=t.map(function(t){return n(t,e,i)});return flatten(a)}return t.evaluate(e,i)}return"function"===r?t(e,i):t},this.makeItems=function(t,e,i,r,n){n()}},FW.Scraper=function(t){FW._scrapers.push(new FW._Scraper(t))},FW._Scraper=function(t){for(x in t)this[x]=t[x];this._singleFieldNames=["abstractNote","applicationNumber","archive","archiveLocation","artworkMedium","artworkSize","assignee","audioFileType","audioRecordingType","billNumber","blogTitle","bookTitle","callNumber","caseName","code","codeNumber","codePages","codeVolume","committee","company","conferenceName","country","court","date","dateDecided","dateEnacted","dictionaryTitle","distributor","docketNumber","documentNumber","DOI","edition","encyclopediaTitle","episodeNumber","extra","filingDate","firstPage","forumTitle","genre","history","institution","interviewMedium","ISBN","ISSN","issue","issueDate","issuingAuthority","journalAbbreviation","label","language","legalStatus","legislativeBody","letterType","libraryCatalog","manuscriptType","mapType","medium","meetingName","nameOfAct","network","number","numberOfVolumes","numPages","pages","patentNumber","place","postType","presentationType","priorityNumbers","proceedingsTitle","programTitle","programmingLanguage","publicLawNumber","publicationTitle","publisher","references","reportNumber","reportType","reporter","reporterVolume","rights","runningTime","scale","section","series","seriesNumber","seriesText","seriesTitle","session","shortTitle","studio","subject","system","thesisType","title","type","university","url","version","videoRecordingType","volume","websiteTitle","websiteType"],this._makeAttachments=function(t,e,i,r){if(i instanceof Array)i.forEach(function(i){this._makeAttachments(t,e,i,r)},this);else if("object"==typeof i){var n=i.urls||i.url,a=i.types||i.type,s=i.titles||i.title,o=i.snapshots||i.snapshot,u=this.evaluateThing(n,t,e),l=this.evaluateThing(s,t,e),c=this.evaluateThing(a,t,e),h=this.evaluateThing(o,t,e);u instanceof Array||(u=[u]);for(var f in u){var p,m,v,d=u[f];p=c instanceof Array?c[f]:c,m=l instanceof Array?l[f]:l,v=h instanceof Array?h[f]:h,r.attachments.push({url:d,title:m,mimeType:p,snapshot:v})}}},this.makeItems=function(t,e,i,r,n){var a=new Zotero.Item(this.itemType);a.url=e;for(var s in this._singleFieldNames){var o=this._singleFieldNames[s];if(this[o]){var u=this.evaluateThing(this[o],t,e);u instanceof Array?a[o]=u[0]:a[o]=u}}var l=["creators","tags"];for(var c in l){var h=l[c],f=this.evaluateThing(this[h],t,e);if(f)for(var p in f)a[h].push(f[p])}this._makeAttachments(t,e,this.attachments,a),r(a,this,t,e),n()}},FW._Scraper.prototype=new FW._Base,FW.MultiScraper=function(t){FW._scrapers.push(new FW._MultiScraper(t))},FW._MultiScraper=function(t){for(x in t)this[x]=t[x];this._mkSelectItems=function(t,e){var i=new Object;for(var r in t)i[e[r]]=t[r];return i},this._selectItems=function(t,e,i){var r=new Array;Zotero.selectItems(this._mkSelectItems(t,e),function(t){for(var e in t)r.push(e);i(r)})},this._mkAttachments=function(t,e,i){var r=this.evaluateThing(this.attachments,t,e),n=new Object;if(r)for(var a in i)n[i[a]]=r[a];return n},this._makeChoices=function(t,e,i,r,n){if(t instanceof Array)t.forEach(function(t){this._makeTitlesUrls(t,e,i,r,n)},this);else if("object"==typeof t){var a=t.urls||t.url,s=t.titles||t.title,o=this.evaluateThing(a,e,i),u=this.evaluateThing(s,e,i),l=u instanceof Array;o instanceof Array||(o=[o]);for(var c in o){var h,f=o[c];h=l?u[c]:u,n.push(f),r.push(h)}}},this.makeItems=function(t,e,i,r,n){if(this.beforeFilter){var a=this.beforeFilter(t,e);if(a!=e)return void this.makeItems(t,a,i,r,n)}var s=[],o=[];this._makeChoices(this.choices,t,e,s,o);var u=this._mkAttachments(t,e,o),l=this.itemTrans;this._selectItems(s,o,function(t){if(t){var e=function(t){var e=t.documentURI,i=l;void 0===i&&(i=FW.getScraper(t,e)),void 0===i||i.makeItems(t,e,u[e],r,function(){})};Zotero.Utilities.processDocuments(t,e,n)}else n()})}},FW._MultiScraper.prototype=new FW._Base,FW.WebDelegateTranslator=function(t){return new FW._WebDelegateTranslator(t)},FW._WebDelegateTranslator=function(t){for(x in t)this[x]=t[x];this.makeItems=function(t,e,i,r,n){var a=this,s=Zotero.loadTranslator("web");s.setHandler("itemDone",function(i,n){r(n,a,t,e)}),s.setDocument(t),this.translatorId?(s.setTranslator(this.translatorId),s.translate()):(s.setHandler("translators",function(t,e){e.length&&(s.setTranslator(e[0]),s.translate())}),s.getTranslators()),n()}},FW._WebDelegateTranslator.prototype=new FW._Base,FW._StringMagic=function(){this._filters=new Array,this.addFilter=function(t){return this._filters.push(t),this},this.split=function(t){return this.addFilter(function(e){return e.split(t).filter(function(t){return""!=t})})},this.replace=function(t,e,i){return this.addFilter(function(r){return r.match(t)?r.replace(t,e,i):r})},this.prepend=function(t){return this.replace(/^/,t)},this.append=function(t){return this.replace(/$/,t)},this.remove=function(t,e){return this.replace(t,"",e)},this.trim=function(){return this.addFilter(function(t){return Zotero.Utilities.trim(t)})},this.trimInternal=function(){return this.addFilter(function(t){return Zotero.Utilities.trimInternal(t)})},this.match=function(t,e){return e||(e=0),this.addFilter(function(i){var r=i.match(t);return void 0===r||null===r?void 0:r[e]})},this.cleanAuthor=function(t,e){return this.addFilter(function(i){return Zotero.Utilities.cleanAuthor(i,t,e)})},this.key=function(t){return this.addFilter(function(e){return e[t]})},this.capitalizeTitle=function(){return this.addFilter(function(t){return Zotero.Utilities.capitalizeTitle(t)})},this.unescapeHTML=function(){return this.addFilter(function(t){return Zotero.Utilities.unescapeHTML(t)})},this.unescape=function(){return this.addFilter(function(t){return unescape(t)})},this._applyFilters=function(t,e){for(i in this._filters){t=flatten(t),t=t.filter(function(t){return void 0!==t&&null!==t});for(var r=0;r0&&a[0])return n}},FW.getScraper=function(t,e){var i=FW.detectWeb(t,e);return FW._scrapers.filter(function(r){return r.evaluateThing(r.itemType,t,e)==i&&r.evaluateThing(r.detect,t,e)})[0]},FW.doWeb=function(t,e){var i=FW.getScraper(t,e);i.makeItems(t,e,[],function(t,e,i,r){e.callHook("scraperDone",t,i,r),t.title||(t.title=""),t.complete()},function(){Zotero.done()}),Zotero.wait()}; -function detectWeb(doc, url) { return FW.detectWeb(doc, url); } -function doWeb(doc, url) { return FW.doWeb(doc, url); } - - - /* ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2012/2013 Sebastian Karcher - + + Copyright © 2012/2013 Sebastian Karcher, Zoë C. Ma, and contributors + This file is part of Zotero. - + Zotero is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Zotero is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. - + You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - + along with Zotero. If not, see . + ***** END LICENSE BLOCK ***** */ -/**ToC **/ -FW.MultiScraper({ -itemType : 'multiple', -detect : FW.Xpath('//h6/a'), -choices : { - titles : FW.Xpath('//h6/a|//h2/a').text().trim(), - urls : FW.Xpath('//h6/a|//h2/a').key("href") + +function detectWeb(doc, url) { + let pageURL = new URL(url); + // Some content may not be in the print edition, but the web publication + // is a magazine nonetheless + if (/^\/(articles|forum(_response)?|us)\/.+/.test(pageURL.pathname)) { + return 'magazineArticle'; + } + else if (getSearchResults(doc, true)) { + return 'multiple'; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + let outerSelectors = [ + "[data-elementor-type='search-results']", // search page + "[data-elementor-post-type='issue']", // issue TOC + "[data-elementor-post-type='special-project']", // special project + ]; + // Titles are in an "h3 a" inside the outer container element + // The computed selector looks like "container1 h3 a, container2 h3 a ..." + let selectors = outerSelectors.map(s => s + " h3 a").join(", "); + // Legacy issue pages; although many of the links are broken + selectors += ", [data-elementor-post-type='elementor_library'] h6 a"; + var rows = doc.querySelectorAll(selectors); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; } -}); - -/**Search Results**/ -FW.MultiScraper({ -itemType : 'multiple', -detect : FW.Xpath('//li[@class="search-result"]//h3/a'), -choices : { - titles : FW.Xpath('//h3/a').text().trim(), - urls : FW.Xpath('//h3/a').key("href") + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } } -}); - -/** Articles */ -FW.Scraper({ -itemType : 'magazineArticle', -detect : FW.Xpath('//div[@class="title"]/h1'), -title : FW.Xpath('//div[@class="title"]/h1').text().trim(), -attachments : [{ url: FW.Url(), - title: "Boston Review Snapshot", - type: "text/html" }], -creators : FW.Xpath('//div[@class="article-author"]//div[@class="author-name"]'+ - '|//div[contains(@class, "field-name-field-author")]//a').text().split(/ and /).cleanAuthor("author"), -date : FW.Xpath('//span[@class="date-display-single"]/@content').text(), -ISSN : "0734-2306", -abstractNote : FW.Xpath('//div[contains(@class, "field-name-field-subhead")]').text(), -publicationTitle : "Boston Review", -language : "en-US" -}); - +async function scrape(doc, url = doc.location.href) { + let translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + + translator.setHandler('itemDone', (_obj, item) => { + item.publicationTitle = item.libraryCatalog = "Boston Review"; + let ldInfo = text(doc, "script[type='application/ld+json']"); + let date; + if (ldInfo) { + // Get schema.org metadata for the web page from JSON-LD + let pageInfo + = (JSON.parse(ldInfo)["@graph"] || []) + .filter(x => x["@type"] === "WebPage")[0]; + date = pageInfo && pageInfo.datePublished; + } + if (!date) { + // Only as fallback; despite the itemprop value, this doesn't + // always appear to be "date modified"; rather, it's the original + // publication date + date = text(doc, ".elementor-post-info [itemprop='dateModified']"); + } + if (date) { + item.date = ZU.strToISO(date); + } + + // Remove suffix " - Boston Review" in title + item.title = item.title.replace(/\s+-\s+Boston Review\s*$/, ""); + + // NOTE: the href property match takes care of both /author and + // /author-custom paths + let authors = doc.querySelectorAll("h2 a[href^='https://www.bostonreview.net/author']"); + for (let author of authors) { + let authorName = ZU.trimInternal(author.textContent.trim()); + item.creators.push(ZU.cleanAuthor(authorName, "author")); + } + + for (let tag of doc.querySelectorAll("a[href^='https://www.bostonreview.net/tag/']")) { + item.tags.push(ZU.trimInternal(tag.textContent)); + } + + // NOTE that in general there's no sure way to determine whether an + // article belongs to a print issue (hence no volume/issue numbers and + // no ISSN which is for the print publication). If you can help, please + // contribute! + + item.complete(); + }); + + let em = await translator.getTranslatorObject(); + em.itemType = 'magazineArticle'; + await em.doWeb(doc, url); +} /** BEGIN TEST CASES **/ var testCases = [ { "type": "web", - "url": "http://www.bostonreview.net/forum/promoting-social-mobility/rethinking-family-life-robin-west", + "url": "https://www.bostonreview.net/forum_response/rethinking-family-life-robin-west/", "items": [ { "itemType": "magazineArticle", - "title": "Promoting Social Mobility", + "title": "Rethinking Family Life", "creators": [ { "firstName": "Robin", @@ -96,15 +158,15 @@ var testCases = [ "creatorType": "author" } ], - "date": "2012-09-01T00:00:00-04:00", - "ISSN": "0734-2306", + "date": "2012-11-09", + "abstractNote": "Rethinking Family Life James Heckman provides an economic argument for a claim that is often thought to be supported at most by moral considerations:", "language": "en-US", "libraryCatalog": "Boston Review", "publicationTitle": "Boston Review", - "url": "http://www.bostonreview.net/forum/promoting-social-mobility/rethinking-family-life-robin-west", + "url": "https://www.bostonreview.net/forum_response/rethinking-family-life-robin-west/", "attachments": [ { - "title": "Boston Review Snapshot", + "title": "Snapshot", "mimeType": "text/html" } ], @@ -116,7 +178,7 @@ var testCases = [ }, { "type": "web", - "url": "http://www.bostonreview.net/forum/can-global-brands-create-just-supply-chains-richard-locke", + "url": "https://www.bostonreview.net/forum/can-global-brands-create-just-supply-chains-richard-locke", "items": [ { "itemType": "magazineArticle", @@ -128,19 +190,29 @@ var testCases = [ "creatorType": "author" } ], - "date": "2013-05-21T00:00:00-04:00", - "ISSN": "0734-2306", + "date": "2013-05-21", + "abstractNote": "When Jia Jingchuan, a 27-year-old electronics worker in Suzhou, China, sought compensation for the chemical poisoning he suffered at work, he appealed", "language": "en-US", "libraryCatalog": "Boston Review", "publicationTitle": "Boston Review", - "url": "http://www.bostonreview.net/forum/can-global-brands-create-just-supply-chains-richard-locke", + "url": "https://www.bostonreview.net/forum/can-global-brands-create-just-supply-chains-richard-locke/", "attachments": [ { - "title": "Boston Review Snapshot", + "title": "Snapshot", "mimeType": "text/html" } ], - "tags": [], + "tags": [ + { + "tag": "Global" + }, + { + "tag": "Human Rights" + }, + { + "tag": "Labor" + } + ], "notes": [], "seeAlso": [] } @@ -148,7 +220,7 @@ var testCases = [ }, { "type": "web", - "url": "http://www.bostonreview.net/us/government-loansharking", + "url": "https://www.bostonreview.net/articles/government-loansharking/", "items": [ { "itemType": "magazineArticle", @@ -160,20 +232,32 @@ var testCases = [ "creatorType": "author" } ], - "date": "2013-06-07T00:00:00-04:00", - "ISSN": "0734-2306", - "abstractNote": "An Update on the Student-Debt Crisis", + "date": "2013-06-07", + "abstractNote": "Last November when I first wrote about student loans for Boston Review, the Department of Education estimated it would be pulling in around $25 billion in", "language": "en-US", "libraryCatalog": "Boston Review", "publicationTitle": "Boston Review", - "url": "http://www.bostonreview.net/us/government-loansharking", + "url": "https://www.bostonreview.net/articles/government-loansharking/", "attachments": [ { - "title": "Boston Review Snapshot", + "title": "Snapshot", "mimeType": "text/html" } ], - "tags": [], + "tags": [ + { + "tag": "Economy" + }, + { + "tag": "Education" + }, + { + "tag": "Politics" + }, + { + "tag": "U.S." + } + ], "notes": [], "seeAlso": [] } @@ -181,13 +265,68 @@ var testCases = [ }, { "type": "web", - "url": "http://bostonreview.net/septemberoctober-2012", + "url": "https://bostonreview.net/issue/september-october-2012/", "items": "multiple" }, { "type": "web", - "url": "http://www.bostonreview.net/search/node/labor", + "url": "https://www.bostonreview.net/?s=labor", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.bostonreview.net/articles/astra-taylor-wolson-interview/", + "items": [ + { + "itemType": "magazineArticle", + "title": "Beyond the Neoliberal University", + "creators": [ + { + "firstName": "Astra", + "lastName": "Taylor", + "creatorType": "author" + }, + { + "firstName": "Todd", + "lastName": "Wolfson", + "creatorType": "author" + } + ], + "date": "2020-08-04", + "abstractNote": "Astra Taylor talks with Rutgers faculty union president Todd Wolfson about organizing academic communities in the age of COVID-19.", + "language": "en-US", + "libraryCatalog": "Boston Review", + "publicationTitle": "Boston Review", + "url": "https://www.bostonreview.net/articles/astra-taylor-wolson-interview/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "COVID-19" + }, + { + "tag": "Education" + }, + { + "tag": "Interview" + }, + { + "tag": "Labor" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.bostonreview.net/special_project/opportunity-after-neoliberalism/", "items": "multiple" } ] -/** END TEST CASES **/ \ No newline at end of file +/** END TEST CASES **/ diff --git a/Bosworth Toller's Anglo-Saxon Dictionary Online.js b/Bosworth Toller's Anglo-Saxon Dictionary Online.js new file mode 100644 index 00000000000..34f78a29a27 --- /dev/null +++ b/Bosworth Toller's Anglo-Saxon Dictionary Online.js @@ -0,0 +1,496 @@ +{ + "translatorID": "b2d07a2a-c8c6-4426-ba6b-35f094a4d916", + "label": "Bosworth Toller's Anglo-Saxon Dictionary Online", + "creator": "Zoë C. Ma", + "target": "^https://bosworthtoller\\.com/", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-08-18 07:39:58" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Zoë C. Ma + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + + +function detectWeb(doc) { + // The logic is that "a page's type is determined by its content", because + // the current implementation of the dictionary web app may not be able to + // sync URL correctly all the time. + if (getSearchResults(doc, true/* checkOnly */)) { + return "multiple"; + } + + if (doc.querySelector("#btd--entry-single")) { + return "dictionaryEntry"; + } + + return false; +} + +function getSearchResults(doc, checkOnly = false) { + let items = {}; + let found = false; + let rows = doc.querySelectorAll(".btd--search-entry"); + for (let row of rows) { + // Don't retrieve the "similar entry" links + let href = attr(row, ".btd--search-entry-header a", "href"); + let title = ZU.trimInternal(text(row, ".btd--entry-grammar").trim()); + if (!title) { + title = text(row, ".btd--search-entry-header a"); // fallback + } + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc) === 'multiple') { + let items = await Z.selectItems(getSearchResults(doc)); + if (!items) return; + for (let url of Object.keys(items)) { + scrape(await requestDocument(url)); + } + } + else { + scrape(doc, url); + } +} + +const BOSWORTH_TOLLER_INFO = { + dictionaryTitle: "An Anglo-Saxon Dictionary Online", + language: "en", + place: "Prague", + publisher: "Faculty of Arts, Charles University", + date: "2014", + creators: [ + { firstName: "Joseph", lastName: "Bosworth", creatorType: "author" }, + { firstName: "Thomas Northcote", lastName: "Toller", creatorType: "editor" }, + { firstName: "Christ", lastName: "Sean", creatorType: "editor" }, + { firstName: "Ondřej", lastName: "Tichy", creatorType: "editor" }, + ], +}; + +function scrape(doc, url = doc.location.href) { + let item = new Z.Item("dictionaryEntry"); + + // "Constant" fields + Object.assign(item, BOSWORTH_TOLLER_INFO); + + // Page-specific data + item.url = url; + + // Word entry + item.title = normalizeLemma(doc) || "[Unknown entry]"; + + // Original publication and page number in it, if any, as extra + item.extra = getExtraInfo(doc); + + // Snapshot + item.attachments = [{ + document: doc, + title: "Snapshot", + mimeType: "text/html" + }]; + + item.complete(); +} + +// See https://bosworthtoller.com/images-dictionary/frontback_matter.pdf +var BOOK_ORIG_INFO = { + b: "Original Dictionary Title: An Anglo-Saxon Dictionary, Based on the Manuscript Collections of the Late Joseph Bosworth, D.D., F.R.S.\nOriginal Date: 1898\nOriginal Publisher: Oxford University Press\nOriginal Place: London", + d: "Original Dictionary Title: An Anglo-Saxon Dictionary, Based on the Manuscript Collections of the Late Joseph Bosworth; Supplement\nOriginal Date: 1921\nOriginal Publisher: Oxford University Press\nOriginal Place: London", +}; + +// Get the extra info including original volume info and page number by parsing +// the URL of the scanned page linked to the article. Returns a string where +// each extra entry occupies one line in `key: value` format, or empty string +// if the original book and page cannot be determined. +function getExtraInfo(doc) { + let imageURL = attr(doc, ".btd--image-pin-pan > img", "src"); + if (!imageURL) { + return ""; + } + // "b" for main book (1898), "d" for supplement (1912) + let pageMatch = imageURL.match(/^\/images-dictionary\/bt_([bd])(\d+)\..+$/); + if (!pageMatch) { + return ""; + } + let [, bookKey, page] = pageMatch; + return BOOK_ORIG_INFO[bookKey] // static original publication info + + "\nOriginal Page: " + page.replace(/^0*/, ""); // trim leading zero +} + +// Normalize the lemma's vowel display-form, following the original book's +// orthography (acute for long vowel). +// Why is this necessary? Because the lemma prominently displayed on the page +// main body can be configured by the user (by clicking on the icons: acute, +// macron, and none). But we want the form used in our item data normalized, no +// matter the display option, in order to not lose information and avoid +// duplication. +// NOTE that the letter case is not normalized -- the display on the page +// corresponds to the lemma in the original book. +function normalizeLemma(doc) { + // The key is to apply the correct vowel length even if the user disables + // its display. This "canonical" form (which corresponds to the original + // form in the print book) can be found either in the metadata or in the + // "citation" block under "entry information" in the doc, no matter the + // citation style in use on the page. + let titleRaw = text(doc, "#btd--entry-lemma").trim(); + let titleNormalized = removeDiacritics(titleRaw); + + // lemma from the meta field, not normally amenable to client-side + // modification + let metaTitle = attr(doc, 'meta[property="og:title"]', "content").trim(); + let metaTitleNormalized = removeDiacritics(metaTitle); + + if (metaTitle && titleNormalized === metaTitleNormalized) { + return metaTitle; + } + + return null; +} + +// Utility functions + +// Remove the acute accent and macron if any. +function removeDiacritics(str) { + return str.normalize("NFD").replace(/[\u0301\u0304]/g, ""); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://bosworthtoller.com/search?q=heorte", + "defer": true, + "items": "multiple" + }, + { + "type": "web", + "url": "https://bosworthtoller.com/search/advanced?q=%7B%22minFields%22%3A1,%22fields%22%3A%5B%7B%22query%22%3A%22diacon%22,%22field%22%3A%22headword%22,%22is_regex%22%3Afalse%7D,%7B%22query%22%3A%22%22,%22field%22%3A%22headword%22,%22condition%22%3A%22and%22,%22is_regex%22%3Afalse%7D%5D,%22wordclass%22%3A%7B%22include%22%3A%5B%221%22%5D,%22exclude%22%3A%5B%5D%7D,%22gender%22%3A%7B%22include%22%3A%5B%221%22%5D,%22exclude%22%3A%5B%5D%7D,%22subcategory%22%3A%7B%22include%22%3A%5B%5D,%22exclude%22%3A%5B%5D%7D,%22volume%22%3Anull%7D", + "defer": true, + "items": "multiple" + }, + { + "type": "web", + "url": "https://bosworthtoller.com/23205", + "items": [ + { + "itemType": "dictionaryEntry", + "title": "mucg-wyrt", + "creators": [ + { + "firstName": "Joseph", + "lastName": "Bosworth", + "creatorType": "author" + }, + { + "firstName": "Thomas Northcote", + "lastName": "Toller", + "creatorType": "editor" + }, + { + "firstName": "Christ", + "lastName": "Sean", + "creatorType": "editor" + }, + { + "firstName": "Ondřej", + "lastName": "Tichy", + "creatorType": "editor" + } + ], + "date": "2014", + "dictionaryTitle": "An Anglo-Saxon Dictionary Online", + "extra": "Original Dictionary Title: An Anglo-Saxon Dictionary, Based on the Manuscript Collections of the Late Joseph Bosworth, D.D., F.R.S.\nOriginal Date: 1898\nOriginal Publisher: Oxford University Press\nOriginal Place: London\nOriginal Page: 700", + "language": "en", + "libraryCatalog": "Bosworth Toller's Anglo-Saxon Dictionary Online", + "place": "Prague", + "publisher": "Faculty of Arts, Charles University", + "url": "https://bosworthtoller.com/23205", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://bosworthtoller.com/7096", + "items": [ + { + "itemType": "dictionaryEntry", + "title": "CYN", + "creators": [ + { + "firstName": "Joseph", + "lastName": "Bosworth", + "creatorType": "author" + }, + { + "firstName": "Thomas Northcote", + "lastName": "Toller", + "creatorType": "editor" + }, + { + "firstName": "Christ", + "lastName": "Sean", + "creatorType": "editor" + }, + { + "firstName": "Ondřej", + "lastName": "Tichy", + "creatorType": "editor" + } + ], + "date": "2014", + "dictionaryTitle": "An Anglo-Saxon Dictionary Online", + "extra": "Original Dictionary Title: An Anglo-Saxon Dictionary, Based on the Manuscript Collections of the Late Joseph Bosworth, D.D., F.R.S.\nOriginal Date: 1898\nOriginal Publisher: Oxford University Press\nOriginal Place: London\nOriginal Page: 183", + "language": "en", + "libraryCatalog": "Bosworth Toller's Anglo-Saxon Dictionary Online", + "place": "Prague", + "publisher": "Faculty of Arts, Charles University", + "url": "https://bosworthtoller.com/7096", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://bosworthtoller.com/27305", + "items": [ + { + "itemType": "dictionaryEntry", + "title": "secgan", + "creators": [ + { + "firstName": "Joseph", + "lastName": "Bosworth", + "creatorType": "author" + }, + { + "firstName": "Thomas Northcote", + "lastName": "Toller", + "creatorType": "editor" + }, + { + "firstName": "Christ", + "lastName": "Sean", + "creatorType": "editor" + }, + { + "firstName": "Ondřej", + "lastName": "Tichy", + "creatorType": "editor" + } + ], + "date": "2014", + "dictionaryTitle": "An Anglo-Saxon Dictionary Online", + "extra": "Original Dictionary Title: An Anglo-Saxon Dictionary, Based on the Manuscript Collections of the Late Joseph Bosworth, D.D., F.R.S.\nOriginal Date: 1898\nOriginal Publisher: Oxford University Press\nOriginal Place: London\nOriginal Page: 855", + "language": "en", + "libraryCatalog": "Bosworth Toller's Anglo-Saxon Dictionary Online", + "place": "Prague", + "publisher": "Faculty of Arts, Charles University", + "url": "https://bosworthtoller.com/27305", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://bosworthtoller.com/23035", + "items": [ + { + "itemType": "dictionaryEntry", + "title": "mód-c-wánig", + "creators": [ + { + "firstName": "Joseph", + "lastName": "Bosworth", + "creatorType": "author" + }, + { + "firstName": "Thomas Northcote", + "lastName": "Toller", + "creatorType": "editor" + }, + { + "firstName": "Christ", + "lastName": "Sean", + "creatorType": "editor" + }, + { + "firstName": "Ondřej", + "lastName": "Tichy", + "creatorType": "editor" + } + ], + "date": "2014", + "dictionaryTitle": "An Anglo-Saxon Dictionary Online", + "extra": "Original Dictionary Title: An Anglo-Saxon Dictionary, Based on the Manuscript Collections of the Late Joseph Bosworth, D.D., F.R.S.\nOriginal Date: 1898\nOriginal Publisher: Oxford University Press\nOriginal Place: London\nOriginal Page: 694", + "language": "en", + "libraryCatalog": "Bosworth Toller's Anglo-Saxon Dictionary Online", + "place": "Prague", + "publisher": "Faculty of Arts, Charles University", + "url": "https://bosworthtoller.com/23035", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://bosworthtoller.com/53107", + "items": [ + { + "itemType": "dictionaryEntry", + "title": "hrǽw", + "creators": [ + { + "firstName": "Joseph", + "lastName": "Bosworth", + "creatorType": "author" + }, + { + "firstName": "Thomas Northcote", + "lastName": "Toller", + "creatorType": "editor" + }, + { + "firstName": "Christ", + "lastName": "Sean", + "creatorType": "editor" + }, + { + "firstName": "Ondřej", + "lastName": "Tichy", + "creatorType": "editor" + } + ], + "date": "2014", + "dictionaryTitle": "An Anglo-Saxon Dictionary Online", + "extra": "Original Dictionary Title: An Anglo-Saxon Dictionary, Based on the Manuscript Collections of the Late Joseph Bosworth; Supplement\nOriginal Date: 1921\nOriginal Publisher: Oxford University Press\nOriginal Place: London\nOriginal Page: 562", + "language": "en", + "libraryCatalog": "Bosworth Toller's Anglo-Saxon Dictionary Online", + "place": "Prague", + "publisher": "Faculty of Arts, Charles University", + "url": "https://bosworthtoller.com/53107", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://bosworthtoller.com/42878", + "items": [ + { + "itemType": "dictionaryEntry", + "title": "dón", + "creators": [ + { + "firstName": "Joseph", + "lastName": "Bosworth", + "creatorType": "author" + }, + { + "firstName": "Thomas Northcote", + "lastName": "Toller", + "creatorType": "editor" + }, + { + "firstName": "Christ", + "lastName": "Sean", + "creatorType": "editor" + }, + { + "firstName": "Ondřej", + "lastName": "Tichy", + "creatorType": "editor" + } + ], + "date": "2014", + "dictionaryTitle": "An Anglo-Saxon Dictionary Online", + "extra": "Original Dictionary Title: An Anglo-Saxon Dictionary, Based on the Manuscript Collections of the Late Joseph Bosworth; Supplement\nOriginal Date: 1921\nOriginal Publisher: Oxford University Press\nOriginal Place: London\nOriginal Page: 154", + "language": "en", + "libraryCatalog": "Bosworth Toller's Anglo-Saxon Dictionary Online", + "place": "Prague", + "publisher": "Faculty of Arts, Charles University", + "url": "https://bosworthtoller.com/42878", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/Brill.js b/Brill.js index 8234ed1c7a0..dc23091531e 100644 --- a/Brill.js +++ b/Brill.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-10-21 04:55:10" + "lastUpdated": "2023-08-17 20:03:38" } /* @@ -86,19 +86,10 @@ function doWeb(doc, url) { function scrape(doc, url) { if (url.includes('bibliographies.brillonline.com/entries/')) { - scrapeBibliography(doc, url); + scrapeBibliography(doc); return; } - if (doc.querySelector('body > meta')) { - // Brill's HTML is structured incorrectly, and it causes some parsers - // to interpret the tags as being in the body, which breaks EM. - // We'll fix it here. - for (let meta of doc.querySelectorAll('body > meta')) { - doc.head.appendChild(meta); - } - } - var translator = Zotero.loadTranslator('web'); // Embedded Metadata translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); @@ -140,12 +131,17 @@ function scrape(doc, url) { if (url.includes('referenceworks.brillonline.com/entries/')) { trans.itemType = 'encyclopediaArticle'; } - + + // Brill's HTML is structured incorrectly due to a bug in the + // Pubfactory CMS, and it causes some parsers to put the + // tags in the body. We'll fix it by telling EM to work around it. + trans.searchForMetaTagsInBody = true; + trans.doWeb(doc, url); }); } -function scrapeBibliography(doc, url) { +function scrapeBibliography(doc) { let params = new URLSearchParams({ entryId: attr(doc, 'input[name="entryId"]', 'value'), dest: attr(doc, 'input[name="dest"]', 'value') diff --git a/CNKI.js b/CNKI.js index 9029f6eecb4..61d41acb12f 100644 --- a/CNKI.js +++ b/CNKI.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-03-08 12:33:13" + "lastUpdated": "2023-11-27 05:30:22" } /* @@ -38,47 +38,35 @@ // Fetches RefWorks records for provided IDs and calls onDataAvailable with resulting text // ids should be in the form [{dbname: "CDFDLAST2013", filename: "1013102302.nh"}] -function getRefWorksByID(ids, onDataAvailable) { - if (!ids.length) return; - var { dbname, filename, url } = ids.shift(); - let postData = "filename=" + filename + - "&displaymode=Refworks&orderparam=0&ordertype=desc&selectfield=&dbname=" + - dbname + "&random=0.2111567532240084"; - - ZU.doPost('https://kns.cnki.net/KNS8/manage/ShowExport', postData, - function (text) { - let data = text - .replace("
  • ", "") - .replace("
", "") - .replace("
  • ", "") // divide results - .replace(/
    |\r/g, "\n") - .replace(/vo (\d+)\n/, "VO $1\n") // Divide VO and IS to different line - .replace(/IS 0(\d+)\n/g, "IS $1\n") // Remove leading 0 - .replace(/VO 0(\d+)\n/g, "VO $1\n") - .replace(/\n+/g, "\n") - .replace(/\n([A-Z][A-Z1-9]\s)/g, "
    $1") - .replace(/\n/g, "") - .replace(/
    /g, "\n") - .replace(/\t/g, "") // \t in abstract - .replace( - /^RT\s+Conference Proceeding/gim, - "RT Conference Proceedings" - ) - .replace(/^RT\s+Dissertation\/Thesis/gim, "RT Dissertation") - .replace(/^(A[1-4]|U2)\s*([^\r\n]+)/gm, function (m, tag, authors) { - authors = authors.split(/\s*[;，,]\s*/); // that's a special comma - if (!authors[authors.length - 1].trim()) authors.pop(); - return tag + " " + authors.join("\n" + tag + " "); - }) - .trim(); - // Z.debug(data); - onDataAvailable(data, url); - // If more results, keep going - if (ids.length) { - getRefWorksByID(ids, onDataAvailable); - } - } - ); +function toStdRef(reftext) { + return reftext + .body + .replace("
    • ", "") + .replace("
    ", "") + .replace("
  • ", "") // divide results + .replace(/
    |\r/g, "\n") + .replace(/vo (\d+)\n/, "VO $1\n") // Divide VO and IS to different line + .replace(/IS (\d+)\nvo/, "IS $1\nVO")// Uppercase VO + .replace(/IS 0(\d+)\n/g, "IS $1\n")// Remove leading 0 + .replace(/VO 0(\d+)\n/g, "VO $1\n") + .replace(/\n+/g, "\n") + .replace(/\n([A-Z][A-Z1-9]\s)/g, "
    $1") + .replace(/\n/g, "") + .replace(/
    /g, "\n") + .replace(/(K1 .*[\u4e00-\u9fa5]) ([a-zA-Z])/g, "$1;$2")// cn keywwords and en keywords + .replace(/\t/g, "") // \t in abstract + .replace( + /^RT\s+Conference Proceeding/gim, + "RT Conference Proceedings" + ) + .replace(/^RT\s+Dissertation\/Thesis/gim, "RT Dissertation") + .replace(/^(A[1-4]|U2)\s*([^\r\n]+)/gm, function (m, tag, authors) { + authors = authors.split(/\s*[;，,]\s*/); // that's a special comma + if (!authors[authors.length - 1].trim()) authors.pop(); + return tag + " " + authors.join("\n" + tag + " "); + }) + .replace(/LA 中文;?/g, "LA zh-CN") + .trim(); } function getIDFromURL(url) { @@ -96,7 +84,12 @@ function getIDFromURL(url) { function getIDFromRef(doc, url) { let database = attr(doc, '#paramdbname', 'value'); let filename = attr(doc, '#paramfilename', 'value'); - return { dbname: database, filename: filename, url: url }; + if (database && filename) { + return { dbname: database, filename: filename, url: url }; + } + else { + return false; + } } // Get dbname and filename from the link target on the "take note" button in @@ -107,7 +100,7 @@ function getIDFromRef(doc, url) { // required info. The note-taking button appears more stable across the CNKI // domains. function getIDFromNoteTakerLink(doc, url) { - const noteURLString = doc.querySelector("li.btn-note a").href; + const noteURLString = attr(doc, "li.btn-note a", "href"); if (!noteURLString) return false; const urlParams = new URLSearchParams(new URL(noteURLString).search); @@ -119,6 +112,17 @@ function getIDFromNoteTakerLink(doc, url) { return { dbname: dbnameValue, filename: filenameValue, url: url }; } +function getIDFromSearchRow(row) { + var dbcode = attr(row, "a.icon-collect", "data-dbname"); + var filename = attr(row, "a.icon-collect", "data-filename"); + if (dbcode && filename) { + return { dbcode: dbcode, dbname: dbcode, filename: filename }; + } + else { + return false; + } +} + function getIDFromPage(doc, url) { return getIDFromURL(url) || getIDFromRef(doc, url) @@ -130,14 +134,21 @@ function getTypeFromDBName(dbname) { CJFQ: "journalArticle", CJFD: "journalArticle", CAPJ: "journalArticle", + SJES: "journalArticle", + SJPD: "journalArticle", + SSJD: "journalArticle", CCJD: "journalArticle", + CDMD: "journalArticle", + CYFD: "journalArticle", CDFD: "thesis", CMFD: "thesis", CLKM: "thesis", CCND: "newspaperArticle", CPFD: "conferencePaper", + IPFD: "conferencePaper", + SCPD: "patent" }; - var db = dbname.substr(0, 4).toUpperCase(); + var db = dbname.substring(0, 4).toUpperCase(); if (dbType[db]) { return dbType[db]; } @@ -161,7 +172,6 @@ function getItemsFromSearchResults(doc, url, itemInfo) { links = ZU.xpath(doc, '//table[@class="GridTableContent"]/tbody/tr[./td[2]/a]'); aXpath = './td[2]/a'; } - if (!links.length) { return false; } @@ -171,7 +181,7 @@ function getItemsFromSearchResults(doc, url, itemInfo) { var a = ZU.xpath(links[i], aXpath)[0]; var title = ZU.xpathText(a, './node()[not(name()="SCRIPT")]', null, ''); if (title) title = ZU.trimInternal(title); - var id = getIDFromURL(a.href); + var id = getIDFromURL(a.href) || getIDFromSearchRow(links[i]); // pre-released item can not get ID from URL, try to get ID from element.value if (!id) { var td1 = ZU.xpath(links[i], './td')[0]; @@ -191,6 +201,10 @@ function detectWeb(doc, url) { // Z.debug(doc); var id = getIDFromPage(doc, url); var items = getItemsFromSearchResults(doc, url); + var searchResult = doc.querySelector("#ModuleSearchResult"); + if (searchResult) { + Z.monitorDOMChanges(searchResult, { childList: true, subtree: true }); + } if (id) { return getTypeFromDBName(id.dbname); } @@ -202,99 +216,96 @@ function detectWeb(doc, url) { } } -function doWeb(doc, url) { +async function doWeb(doc, url) { if (detectWeb(doc, url) == "multiple") { var itemInfo = {}; var items = getItemsFromSearchResults(doc, url, itemInfo); - Z.selectItems(items, function (selectedItems) { - if (!selectedItems) return; - - var itemInfoByTitle = {}; - var ids = []; - for (var url in selectedItems) { - ids.push(itemInfo[url].id); - itemInfoByTitle[selectedItems[url]] = itemInfo[url]; - itemInfoByTitle[selectedItems[url]].url = url; + let selectItems = await Z.selectItems(items); + if (selectItems) { + for (let url in selectItems) { + await scrape(itemInfo[url].id, doc, { url: url }); } - scrape(ids, doc, url, itemInfoByTitle); - }); + } } else { - scrape([getIDFromPage(doc, url)], doc, url); + await scrape(getIDFromPage(doc, url), doc); } } -function scrape(ids, doc, url, itemInfo) { - getRefWorksByID(ids, function (text) { - var translator = Z.loadTranslator('import'); - translator.setTranslator('1a3506da-a303-4b0a-a1cd-f216e6138d86'); // RefWorks Tagged - text = text.replace(/IS (\d+)\nvo/, "IS $1\nVO"); - translator.setString(text); - - translator.setHandler('itemDone', function (obj, newItem) { - // split names - for (var i = 0, n = newItem.creators.length; i < n; i++) { - var creator = newItem.creators[i]; - if (creator.firstName) continue; - - var lastSpace = creator.lastName.lastIndexOf(' '); - var lastMiddleDot = creator.lastName.lastIndexOf('·'); - if (creator.lastName.search(/[A-Za-z]/) !== -1 && lastSpace !== -1) { - // western name. split on last space - creator.firstName = creator.lastName.substr(0, lastSpace); - creator.lastName = creator.lastName.substr(lastSpace + 1); - } - else if (lastMiddleDot !== -1) { - // translated western name with · as separator - creator.firstName = creator.lastName.substr(0, lastMiddleDot); - creator.lastName = creator.lastName.substr(lastMiddleDot + 1); - } - else { - // Chinese name. first character is last name, the rest are first name - creator.firstName = creator.lastName.substr(1); - creator.lastName = creator.lastName.charAt(0); - } - } - - if (newItem.abstractNote) { - newItem.abstractNote = newItem.abstractNote.replace(/\s*[\r\n]\s*/g, '\n'); +async function scrape(id, doc, extraData) { + var { dbname, filename } = id; + var postData = `FileName=${dbname}!${filename}!1!0&DisplayMode=Refworks&OrderParam=0&OrderType=desc&SelectField=&PageIndex=1&PageSize=20&language=&uniplatform=NZKPT&random=0.30585230060685187`; + var refer = `https://kns.cnki.net/dm/manage/export.html?filename=${dbname}!${filename}!1!0&displaymode=NEW&uniplatform=NZKPT`; + var reftext = await request( + 'https://kns.cnki.net/dm/api/ShowExport', + { + method: "POST", + body: postData, + headers: { + Referer: refer } + } + ); + var translator = Z.loadTranslator('import'); + translator.setTranslator('1a3506da-a303-4b0a-a1cd-f216e6138d86'); // RefWorks Tagged + translator.setString(toStdRef(reftext)); + + translator.setHandler('itemDone', function (obj, newItem) { + // split names + for (var i = 0, n = newItem.creators.length; i < n; i++) { + var creator = newItem.creators[i]; + if (creator.firstName) continue; - // clean up tags. Remove numbers from end - for (var j = 0, l = newItem.tags.length; j < l; j++) { - newItem.tags[j] = newItem.tags[j].replace(/:\d+$/, ''); + var lastSpace = creator.lastName.lastIndexOf(' '); + var lastMiddleDot = creator.lastName.lastIndexOf('·'); + if (/[A-Za-z]/.test(creator.lastName) && lastSpace !== -1) { + // western name. split on last space + creator.firstName = creator.lastName.substring(0, lastSpace); + creator.lastName = creator.lastName.substring(lastSpace + 1); } - - newItem.title = ZU.trimInternal(newItem.title); - if (itemInfo) { - var info = itemInfo[newItem.title]; - if (!info) { - Z.debug('No item info for "' + newItem.title + '"'); - } - else { - newItem.url = info.url; - } + else if (lastMiddleDot !== -1) { + // translated western name with · as separator + creator.firstName = creator.lastName.substring(0, lastMiddleDot); + creator.lastName = creator.lastName.substring(lastMiddleDot + 1); } else { - newItem.url = url; - } - - // CN 中国刊物编号，非refworks中的callNumber - // CN in CNKI refworks format explains Chinese version of ISSN - if (newItem.callNumber) { - // newItem.extra = 'CN ' + newItem.callNumber; - newItem.callNumber = ""; - } - // don't download PDF/CAJ on searchResult(multiple) - var webType = detectWeb(doc, url); - if (webType && webType != 'multiple') { - newItem.attachments = getAttachments(doc, newItem); + // Chinese name. first character is last name, the rest are first name + creator.firstName = creator.lastName.substring(1); + creator.lastName = creator.lastName.charAt(0); } - newItem.complete(); - }); + } + + if (newItem.abstractNote) { + newItem.abstractNote = newItem.abstractNote.replace(/\s*[\r\n]\s*/g, '\n'); + } + + // clean up tags. Remove numbers from end + for (var j = 0, l = newItem.tags.length; j < l; j++) { + newItem.tags[j] = newItem.tags[j].replace(/:\d+$/, ''); + } - translator.translate(); + newItem.title = ZU.trimInternal(newItem.title); + if (extraData) { + newItem.url = extraData.url; + } + else { + newItem.url = id.url; + } + + // CN 中国刊物编号，非refworks中的callNumber + // CN in CNKI refworks format explains Chinese version of ISSN + if (newItem.callNumber) { + // newItem.extra = 'CN ' + newItem.callNumber; + newItem.callNumber = ""; + } + // don't download PDF/CAJ on searchResult(multiple) + var webType = detectWeb(doc, id.url); + if (webType && webType != 'multiple') { + newItem.attachments = getAttachments(doc, newItem); + } + newItem.complete(); }); + translator.translate(); } // get pdf download link @@ -351,6 +362,7 @@ var testCases = [ { "type": "web", "url": "https://kns.cnki.net/KCMS/detail/detail.aspx?dbcode=CJFQ&dbname=CJFDLAST2015&filename=SPZZ201412003&v=MTU2MzMzcVRyV00xRnJDVVJMS2ZidVptRmkva1ZiL09OajNSZExHNEg5WE5yWTlGWjRSOGVYMUx1eFlTN0RoMVQ=", + "defer": true, "items": [ { "itemType": "journalArticle", @@ -391,29 +403,19 @@ var testCases = [ "ISSN": "1000-8713", "abstractNote": "来自中药的水溶性多糖具有广谱治疗和低毒性特点,是天然药物及保健品研发中的重要组成部分。针对中药多糖结构复杂、难以表征的问题,本文以中药黄芪中的多糖为研究对象,采用\"自下而上\"法完成对黄芪多糖的表征。首先使用部分酸水解方法水解黄芪多糖,分别考察了水解时间、酸浓度和温度的影响。在适宜条件(4 h、1.5mol/L三氟乙酸、80℃)下,黄芪多糖被水解为特征性的寡糖片段。接下来,采用亲水作用色谱与质谱联用对黄芪多糖部分酸水解产物进行分离和结构表征。结果表明,提取得到的黄芪多糖主要为1→4连接线性葡聚糖,水解得到聚合度4~11的葡寡糖。本研究对其他中药多糖的表征具有一定的示范作用。", "issue": "12", - "language": "中文;", + "language": "zh-CN", "libraryCatalog": "CNKI", "pages": "1306-1312", "publicationTitle": "色谱", "url": "https://kns.cnki.net/KCMS/detail/detail.aspx?dbcode=CJFQ&dbname=CJFDLAST2015&filename=SPZZ201412003&v=MTU2MzMzcVRyV00xRnJDVVJMS2ZidVptRmkva1ZiL09OajNSZExHNEg5WE5yWTlGWjRSOGVYMUx1eFlTN0RoMVQ=", "volume": "32", - "attachments": [], - "tags": [ - { - "tag": "Astragalus" - }, - { - "tag": "characterization" - }, - { - "tag": "hydrophilic interaction liquid chromatography(HILIC)mass spectrometry(MS)" - }, + "attachments": [ { - "tag": "partial acid hydrolysis" - }, - { - "tag": "polysaccharides" - }, + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ { "tag": "亲水作用色谱" }, @@ -441,6 +443,7 @@ var testCases = [ { "type": "web", "url": "https://kns.cnki.net/KCMS/detail/detail.aspx?dbcode=CMFD&dbname=CMFD201701&filename=1017045605.nh&v=MDc3ODZPZVorVnZGQ3ZrV3JyT1ZGMjZHYk84RzlmTXFwRWJQSVI4ZVgxTHV4WVM3RGgxVDNxVHJXTTFGckNVUkw=", + "defer": true, "items": [ { "itemType": "thesis", @@ -452,27 +455,15 @@ var testCases = [ "creatorType": "author" } ], - "date": "2015", - "abstractNote": "黄瓜(Cucumis sativus L.)是我国最大的保护地栽培蔬菜作物,也是植物性别发育和维管束运输研究的重要模式植物。黄瓜基因组序列图谱已经构建完成,并且在此基础上又完成了全基因组SSR标记开发和涵盖330万个变异位点变异组图谱,成为黄瓜功能基因研究的重要平台和工具,相关转录组研究也有很多报道,不过共表达网络研究还是空白。本实验以温室型黄瓜9930为研究对象,选取10个不同组织,进行转录组测序,获得10份转录组原始数据。在对原始数据去除接头与低质量读段后,将高质量读段用Tophat2回贴到已经发表的栽培黄瓜基因组序列上。用Cufflinks对回贴后的数据计算FPKM值,获得10份组织的2...", - "language": "中文;", + "date": "2017", + "abstractNote": "黄瓜(Cucumis sativus L.)是我国最大的保护地栽培蔬菜作物,也是植物性别发育和维管束运输研究的重要模式植物。黄瓜基因组序列图谱已经构建完成,并且在此基础上又完成了全基因组SSR标记开发和涵盖330万个变异位点变异组图谱,成为黄瓜功能基因研究的重要平台和工具,相关转录组研究也有很多报道,不过共表达网络研究还是空白。本实验以温室型黄瓜9930为研究对象,选取10个不同组织,进行转录组测序,获得10份转录组原始数据。在对原始数据去除接头与低质量读段后,将高质量读段用Tophat2回贴到已经发表的栽培黄瓜基因组序列上。用Cufflinks对回贴后的数据计算FPKM值,获得10份组织的24274基因的表达量数据。计算结果中的回贴率比较理想,不过有些基因的表达量过低。为了防止表达量低的基因对结果的影响,将10份组织中表达量最大小于5的基因去除,得到16924个基因,进行下一步分析。共表达网络的构建过程是将上步获得的表达量数据,利用R语言中WGCNA(weighted gene co-expression network analysis)包构建共表达网络。结果得到的共表达网络包括1134个模块。这些模块中的基因表达模式类似,可以认为是共表达关系。不过结果中一些模块内基因间相关性同其他模块相比比较低,在分析过程中,将模块中基因相关性平均值低于0.9的模块都去除,最终得到839个模块,一共11,844个基因。共表达的基因因其表达模式类似而聚在一起,这些基因可能与10份组织存在特异性关联。为了计算模块与组织间的相关性,首先要对每个模块进行主成分分析(principle component analysis,PCA),获得特征基因(module eigengene,ME),特征基因可以表示这个模块所有基因共有的表达趋势。通过计算特征基因与组织间的相关性,从而挑选出组织特异性模块,这些模块一共有323个。利用topGO功能富集分析的结果表明这些特异性模块所富集的功能与组织相关。共表达基因在染色体上的物理位置经常是成簇分布的。按照基因间隔小于25kb为标准。分别对839个模块进行分析,结果发现在71个模块中共有220个cluster,这些cluster 一般有2～5个基因,cluster中的基因在功能上也表现出一定的联系。共表达基因可能受到相同的转录调控,这些基因在启动子前2kb可能会存在有相同的motif以供反式作用元...", + "language": "zh-CN", "libraryCatalog": "CNKI", "thesisType": "硕士", "university": "南京农业大学", "url": "https://kns.cnki.net/KCMS/detail/detail.aspx?dbcode=CMFD&dbname=CMFD201701&filename=1017045605.nh&v=MDc3ODZPZVorVnZGQ3ZrV3JyT1ZGMjZHYk84RzlmTXFwRWJQSVI4ZVgxTHV4WVM3RGgxVDNxVHJXTTFGckNVUkw=", "attachments": [], "tags": [ - { - "tag": "co-expression" - }, - { - "tag": "cucumber" - }, - { - "tag": "network" - }, - { - "tag": "transcriptome" - }, { "tag": "共表达" }, @@ -494,6 +485,7 @@ var testCases = [ { "type": "web", "url": "https://kns.cnki.net/kcms/detail/detail.aspx?dbcode=CCJD&dbname=CCJDLAST2&filename=ZKSF202002010&uniplatform=NZKPT&v=RM9dl7WiC7a9v7FVB6ov3OwJSXCWzsWIng_BWXok2rj4YFWz9tZ20FRZxDaeDPCm", + "defer": true, "items": [ { "itemType": "journalArticle", @@ -511,16 +503,21 @@ var testCases = [ } ], "date": "2020", - "abstractNote": "<正>一、简介近来再次对俄罗斯(1993)和西班牙(1995)陪审团审判模式进行介绍的原因有两个方面。第一,在废除传统陪审团审判的情况下,要么采取仅由职业法官组成的法院审理案件,要么由职业法官和审讯顾问合议来判断所有的事实问题、法律问题并作出相应判决,这是一种令人惊闻的倒退。", - "issue": "02", - "language": "中文;", + "abstractNote": "<正>一、简介近来再次对俄罗斯(1993)和西班牙(1995)陪审团审判模式进行介绍的原因有两个方面。第一,在废除传统陪审团审判的情况下,要么采取仅由职业法官组成的法院审理案件,要么由职业法官和审讯顾问合议来判断所有的事实问题、法律问题并作出相应判决,这是一种令人惊闻的倒退。", + "issue": "2", + "language": "zh-CN", "libraryCatalog": "CNKI", "pages": "193-212", "publicationTitle": "司法智库", "shortTitle": "欧洲陪审团制度新发展", "url": "https://kns.cnki.net/kcms/detail/detail.aspx?dbcode=CCJD&dbname=CCJDLAST2&filename=ZKSF202002010&uniplatform=NZKPT&v=RM9dl7WiC7a9v7FVB6ov3OwJSXCWzsWIng_BWXok2rj4YFWz9tZ20FRZxDaeDPCm", "volume": "3", - "attachments": [], + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], "tags": [ { "tag": "俄罗斯" @@ -545,7 +542,8 @@ var testCases = [ }, { "type": "web", - "url": "https://kns.cnki.net/kcms2/article/abstract?v=3uoqIhG8C44YLTlOAiTRKibYlV5Vjs7ioT0BO4yQ4m_mOgeS2ml3UHGnAz_wirMwf-b2NsjH_IkCCqUvvwsK8DOvNyxMAxbu&uniplatform=NZKPT", + "url": "https://kns.cnki.net/kcms2/article/abstract?v=aGn3Ey0ZxcAi0XeGEjt5HeH9QvBBKaMwsES4SuFJjIdiexE2qhU8bX2aGBIHriUe6WrMOFyCz6TIuYJGlA_YQUO9h2FJwGt_gZfkHkLHnqVgNK8uMWo5lKYMqxvBPfO6_0Zy21140lIwEFrUw-cJtw==&uniplatform=NZKPT", + "defer": true, "items": [ { "itemType": "journalArticle", @@ -559,31 +557,21 @@ var testCases = [ ], "date": "2022", "ISSN": "1001-2397", - "abstractNote": "我国绿色产品认证标识制度框架已初步形成。作为一项法律制度,绿色产品标识及认证中形成了两组法律关系:一是就产品认可认证,在行政主体、认证机构与申请人之间构成公私混合的规制关系;二是就绿色产品标识授权使用,在上述法律关系主体间构成的商业许可关系。两组法律关系的搭建,形成了我国绿色产品认证标识制度的基本格局。制度的具体完善路径是将现行同类环保产品认证标识纳入绿色产品标识与绿色属性产品标识的二元框架内,或吸收,或拆解,或由市场逐步淘汰,最终形成统一的绿色产品认证标识体系。在制度构建过程中,对第三方认证机构的规制成为制度有效运行的关键。参考域外经验,我国应当通过强化认证机构的独立性,平衡认证机构与申请人...", - "issue": "06", - "language": "中文;", + "abstractNote": "我国绿色产品认证标识制度框架已初步形成。作为一项法律制度,绿色产品标识及认证中形成了两组法律关系:一是就产品认可认证,在行政主体、认证机构与申请人之间构成公私混合的规制关系;二是就绿色产品标识授权使用,在上述法律关系主体间构成的商业许可关系。两组法律关系的搭建,形成了我国绿色产品认证标识制度的基本格局。制度的具体完善路径是将现行同类环保产品认证标识纳入绿色产品标识与绿色属性产品标识的二元框架内,或吸收,或拆解,或由市场逐步淘汰,最终形成统一的绿色产品认证标识体系。在制度构建过程中,对第三方认证机构的规制成为制度有效运行的关键。参考域外经验,我国应当通过强化认证机构的独立性,平衡认证机构与申请人之间的制约关系,以及通过加强行政监管与社会监督,防止认证权力寻租,充分发挥绿色产品认证标识制度的实践效果。", + "issue": "6", + "language": "zh-CN", "libraryCatalog": "CNKI", "pages": "133-145", "publicationTitle": "现代法学", - "url": "https://kns.cnki.net/kcms2/article/abstract?v=3uoqIhG8C44YLTlOAiTRKibYlV5Vjs7ioT0BO4yQ4m_mOgeS2ml3UHGnAz_wirMwf-b2NsjH_IkCCqUvvwsK8DOvNyxMAxbu&uniplatform=NZKPT", + "url": "https://kns.cnki.net/kcms2/article/abstract?v=aGn3Ey0ZxcAi0XeGEjt5HeH9QvBBKaMwsES4SuFJjIdiexE2qhU8bX2aGBIHriUe6WrMOFyCz6TIuYJGlA_YQUO9h2FJwGt_gZfkHkLHnqVgNK8uMWo5lKYMqxvBPfO6_0Zy21140lIwEFrUw-cJtw==&uniplatform=NZKPT", "volume": "44", - "attachments": [], - "tags": [ - { - "tag": "certification trade mark" - }, + "attachments": [ { - "tag": "green product certification" - }, - { - "tag": "green product identification" - }, - { - "tag": "green products" - }, - { - "tag": "third party certification" - }, + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ { "tag": "第三方认证" }, @@ -607,7 +595,8 @@ var testCases = [ }, { "type": "web", - "url": "https://kns.cnki.net/kcms2/article/abstract?v=ARuSRxW-FQHH_OEY6X72RuJIsrP2RHAQAacVbC9CGuvOv08ETIP-MqQO5E296beGN9e8BXVfYGR6l0qfpFxS9gdAPZ5URHqiAY8WVPwSYoF6MXeqOgFQfX5vrMMS_wZaK3j5TPxvx-nDGPfIMtrXBlDrWr9SVlAl&uniplatform=NZKPT", + "url": "https://kns.cnki.net/kcms2/article/abstract?v=aGn3Ey0ZxcCQMiRSLWzbqHFLmF0YiAvOI33I1RqvSIDdZeLKl7q3QL7ioYjCbxuMHo1CSBSG2LYUjI9r30yPonoox-iGbCfgn-YF7W2h79KqPswOTOxrzPV94p2evWa1-zchF2wLCag2WcjSEGNUdSNYdPlVmcGt&uniplatform=NZKPT", + "defer": true, "items": [ { "itemType": "journalArticle", @@ -621,15 +610,20 @@ var testCases = [ ], "date": "2022", "ISSN": "1671-7287", - "abstractNote": "我国对常规污染物的治理取得了显著成效，但以有毒有害化学物质的生产和使用为主要来源的新污染物的环境风险仍然较为严峻。当前我国相关环境法律法规和标准中缺乏对新污染物环境风险管控的要求，对于现有化学物质的环境风险管控还存在较为严重的不足。未来环境法典中新污染物环境风险管控立法应当坚持风险预防原则，但风险预防原则并不以追求“零风险”为目标。新污染物环境风险管控立法总体上应当遵循“风险筛查→风险评估→风险管控”的思路。环境风险评估应当聚焦于从科学角度评估新污染物对公众健康和生态环境带来的“风险”本身，不考虑与环境风险无关的经济、社会等因素。确定什么是“不合理的风险”,除了科学判断之外，也需要“正当程序”...", - "issue": "05", - "language": "中文;", + "abstractNote": "我国对常规污染物的治理取得了显著成效，但以有毒有害化学物质的生产和使用为主要来源的新污染物的环境风险仍然较为严峻。当前我国相关环境法律法规和标准中缺乏对新污染物环境风险管控的要求，对于现有化学物质的环境风险管控还存在较为严重的不足。未来环境法典中新污染物环境风险管控立法应当坚持风险预防原则，但风险预防原则并不以追求“零风险”为目标。新污染物环境风险管控立法总体上应当遵循“风险筛查→风险评估→风险管控”的思路。环境风险评估应当聚焦于从科学角度评估新污染物对公众健康和生态环境带来的“风险”本身，不考虑与环境风险无关的经济、社会等因素。确定什么是“不合理的风险”,除了科学判断之外，也需要“正当程序”的加持。风险无法确定时，比照“存在不合理风险”进行管控。在选择风险管控措施时，应当考虑新污染物对公众健康和生态环境的影响程度以及经济、社会等因素。对于新化学物质，应当秉承“除非能证明无害，否则都应当进行适当风险管控”的理念。", + "issue": "5", + "language": "zh-CN", "libraryCatalog": "CNKI", "pages": "18-30+115", "publicationTitle": "南京工业大学学报(社会科学版)", - "url": "https://kns.cnki.net/kcms2/article/abstract?v=ARuSRxW-FQHH_OEY6X72RuJIsrP2RHAQAacVbC9CGuvOv08ETIP-MqQO5E296beGN9e8BXVfYGR6l0qfpFxS9gdAPZ5URHqiAY8WVPwSYoF6MXeqOgFQfX5vrMMS_wZaK3j5TPxvx-nDGPfIMtrXBlDrWr9SVlAl&uniplatform=NZKPT", + "url": "https://kns.cnki.net/kcms2/article/abstract?v=aGn3Ey0ZxcCQMiRSLWzbqHFLmF0YiAvOI33I1RqvSIDdZeLKl7q3QL7ioYjCbxuMHo1CSBSG2LYUjI9r30yPonoox-iGbCfgn-YF7W2h79KqPswOTOxrzPV94p2evWa1-zchF2wLCag2WcjSEGNUdSNYdPlVmcGt&uniplatform=NZKPT", "volume": "21", - "attachments": [], + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], "tags": [ { "tag": "新污染物风险管控" @@ -654,11 +648,12 @@ var testCases = [ }, { "type": "web", - "url": "https://kns.cnki.net/kcms/detail/detail.aspx?doi=10.13863/j.issn1001-4454.2022.01.030", + "url": "https://kns.cnki.net/kcms2/article/abstract?v=aGn3Ey0ZxcBuyOSvEQLm_QauzuszuNvOETrZkPfTUVjXy6wyG6-n2nHmyA70y6TC3IN6i68HMAN2clvthsV7F1ypcjao4RepuYmOZSEVhLK8lN1UAkOxmQkqtJdHoHI1N1gKQDPjuaEbdR6APIJ1sA==&uniplatform=NZKPT&language=CHS", + "defer": true, "items": [ { "itemType": "journalArticle", - "title": "Box-Behnken Design-响应面法优化碱水解人参茎叶三醇皂苷制备人参皂苷Rg_2工艺研究", + "title": "Box-Behnken Design-响应面法优化碱水解人参茎叶三醇皂苷制备人参皂苷Rg2工艺研究", "creators": [ { "lastName": "史", @@ -699,21 +694,26 @@ var testCases = [ "date": "2022", "DOI": "10.13863/j.issn1001-4454.2022.01.030", "ISSN": "1001-4454", - "abstractNote": "目的：利用Box-Behnken Design-响应面法优选制备人参皂苷Rg_2的最佳工艺参数。方法：以碱解反应的碱度、温度、时间作为考察因素，人参茎叶三醇皂苷中人参皂苷Rg_2含量作为评价指标，运用Design-Expert 8.0.5b软件对工艺参数进行优化并获得最佳工艺参数。结果：经优化得到碱水解人参茎叶三醇皂苷制备人参皂苷Rg_2的最佳工艺参数：反应碱度7.4%、反应温度187℃、反应时间5 h。验证试验表明，在此工艺参数下可将人参皂苷Rg_2含量提高至9.84%,且工艺稳定。结论：经过优化的工艺可有效提高人参茎叶三醇皂苷中人参皂苷Rg_2含量。", - "issue": "01", - "language": "中文;", + "abstractNote": "目的：利用Box-Behnken Design-响应面法优选制备人参皂苷Rg2的最佳工艺参数。方法：以碱解反应的碱度、温度、时间作为考察因素，人参茎叶三醇皂苷中人参皂苷Rg2含量作为评价指标，运用Design-Expert 8.0.5b软件对工艺参数进行优化并获得最佳工艺参数。结果：经优化得到碱水解人参茎叶三醇皂苷制备人参皂苷Rg2的最佳工艺参数：反应碱度7.4%、反应温度187℃、反应时间5 h。验证试验表明，在此工艺参数下可将人参皂苷Rg2含量提高至9.84%,且工艺稳定。结论：经过优化的工艺可有效提高人参茎叶三醇皂苷中人参皂苷Rg2含量。", + "issue": "1", + "language": "zh-CN", "libraryCatalog": "CNKI", "pages": "173-176", "publicationTitle": "中药材", - "url": "https://kns.cnki.net/kcms/detail/detail.aspx?doi=10.13863/j.issn1001-4454.2022.01.030", + "url": "https://kns.cnki.net/kcms2/article/abstract?v=aGn3Ey0ZxcBuyOSvEQLm_QauzuszuNvOETrZkPfTUVjXy6wyG6-n2nHmyA70y6TC3IN6i68HMAN2clvthsV7F1ypcjao4RepuYmOZSEVhLK8lN1UAkOxmQkqtJdHoHI1N1gKQDPjuaEbdR6APIJ1sA==&uniplatform=NZKPT&language=CHS", "volume": "45", - "attachments": [], + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], "tags": [ { "tag": "Box-Behnken Design-响应面法" }, { - "tag": "人参皂苷Rg_2" + "tag": "人参皂苷Rg2" }, { "tag": "人参茎叶三醇皂苷" diff --git a/COBISS.js b/COBISS.js new file mode 100644 index 00000000000..f23b5a34968 --- /dev/null +++ b/COBISS.js @@ -0,0 +1,1945 @@ +{ + "translatorID": "ceace65b-4daf-4200-a617-a6bf24c75607", + "label": "COBISS", + "creator": "Brendan O'Connell", + "target": "^https?://plus\\.cobiss\\.net/cobiss", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-08-17 18:57:34" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Brendan O'Connell + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { + // single items may end in an id number that is 6 digits or more + var itemIDURL = /\d{6,}$/; + // detailed view of single items ends in /#full + var fullRecordURL = /#full$/; + if (url.match(itemIDURL) || url.match(fullRecordURL)) { + // capture type of material directly from the catalog page, e.g. "undergraduate thesis" + var typeOfMaterial = doc.querySelector("button#add-biblioentry-to-shelf").getAttribute("data-mat-type"); + if (typeOfMaterial) { + // use translateItemType function to translate catalog material type into a Zotero + // item type, e.g "thesis" + var detectItemType = translateItemType(typeOfMaterial); + if (detectItemType) { + return detectItemType; + } + // if a catalog item type isn't contained in the hash in translateItemType function, + // return Zotero item type 'book', which is by far the most common item type in this catalog. + else { + return 'book'; + } + } + } + else if (getSearchResults(doc, true)) { + return 'multiple'; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('a.title.value'); + + for (let row of rows) { + let href = row.href; + let title = row.innerText; + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +function constructRISURL(url) { + // catalog page URL: https://plus.cobiss.net/cobiss/si/sl/bib/107937536 + // RIS URL: https://plus.cobiss.net/cobiss/si/sl/bib/risCit/107937536 + + // capture first part of URL, e.g. https://plus.cobiss.net/cobiss/si/sl/bib/ + const firstRegex = /^(.*?)\/bib\//; + let firstUrl = url.match(firstRegex)[0]; + + // capture item ID, e.g. /92020483 + const secondRegex = /\/([^/]+)$/; + let secondUrl = url.match(secondRegex)[0]; + + // outputs correct RIS URL structure + let risURL = firstUrl + "risCit" + secondUrl; + return risURL; +} + +function constructEnglishURL(url) { + // default catalog page URL: https://plus.cobiss.net/cobiss/si/sl/bib/107937536 + // page with English metadata: https://plus.cobiss.net/cobiss/si/en/bib/107937536 + // most COBISS catalogs follow the format where the language code is two characters e.g. "sl" + // except ones with three languages, e.g.: https://plus.cobiss.net/cobiss/cg/cnr_cyrl/bib/20926212 + // where there are language codes for english, latin montenegrin, and cyrillic montenegrin + const firstPartRegex = /https:\/\/plus\.cobiss\.net\/cobiss\/[a-z]{2}\//; + const endPartRegex = /\/bib\/\S*/; + + const firstPart = url.match(firstPartRegex)[0]; + const endPart = url.match(endPartRegex)[0]; + var englishURL = firstPart + "en" + endPart; + return englishURL; +} + +// in the catalog, too many items are classified in RIS as either BOOK or ELEC, +// including many reports, ebooks, etc, that thus are incorrectly assigned itemType "book" or "webpage" +// when we rely on Zotero RIS translator. This map assigns more accurate itemTypes +// based on "type of material" classification in English catalog, instead of relying on RIS. +// this function also assigns itemType for catalog items with no RIS. +function translateItemType(englishCatalogItemType) { + var catalogItemTypeHash = new Map([ + ['undergraduate thesis', 'thesis'], + ['proceedings', 'conferencePaper'], + ['novel', 'book'], + ['science fiction (prose)', 'book'], + ['book', 'book'], + ['handbook', 'book'], + ['proceedings of conference contributions', 'conferencePaper'], + ['professional monograph', 'report'], + ['scientific monograph', 'book'], + ['textbook', 'book'], + ['e-book', 'book'], + ['picture book', 'book'], + ['treatise, study', 'report'], + ['catalogue', 'book'], + ['master\u0027s thesis', 'thesis'], + ['picture book', 'book'], + ['short stories', 'book'], + ['research report', 'report'], + ['poetry', 'book'], + ['dissertation', 'thesis'], + ['picture book', 'book'], + ['offprint', 'magazineArticle'], + ['guide-book', 'book'], + ['expertise', 'hearing'], // court testimony, e.g. https://plus.cobiss.net/cobiss/si/en/bib/94791683 + ['profess. monogr', 'report'], + ['project documentation', 'report'], + ['antiquarian material', 'book'], // mostly books, e.g. https://plus.cobiss.net/cobiss/si/en/bib/7543093 + ['other lit.forms', 'book'], + ['drama', 'book'], + ['strip cartoon', 'book'], + ['documentary lit', 'book'], + ['encyclopedia', 'book'], + ['exercise book', 'book'], + ['educational material', 'book'], + ['review', 'report'], + ['statistics', 'report'], + ['legislation', 'statute'], + ['essay', 'book'], + ['final paper', 'thesis'], + ['standard', 'book'], + ['specialist thesis', 'book'], + ['aphorisms, proverbs', 'book'], + ['humour, satire, parody', 'book'], + ['examin. paper', 'report'], + ['annual', 'report'], + ['yearly', 'report'], + ['documentary lit', 'book'], + ['folk literature', 'book'], + ['patent', 'patent'], + ['regulations', 'report'], + ['conf. materials', 'conferencePaper'], + ['radio play', 'book'], + ['letters', 'book'], + ['literature survey/review', 'report'], + ['statute', 'statute'], + ['matura paper', 'thesis'], + ['seminar paper', 'thesis'], + ['habilitation', 'thesis'], + ['dramaturgical paper', 'thesis'], + ['article, component part', 'journalArticle'], + ['e-article', 'journalArticle'], + ['periodical', 'book'], + ['monogr. series', 'book'], + ['audio CD', 'audioRecording'], + ['audio cassette', 'audioRecording'], + ['disc', 'audioRecording'], + ['music, sound recording', 'audioRecording'], + ['audio DVD', 'audioRecording'], + ['printed and manuscript music', 'audioRecording'], + ['graphics', 'artwork'], + ['poster', 'artwork'], + ['photograph', 'artwork'], + ['e-video', 'videoRecording'], + ['video DVD', 'videoRecording'], + ['video cassette', 'videoRecording'], + ['blu-ray', 'videoRecording'], + ['motion picture', 'videoRecording'], + ['map', 'map'], + ['atlas', 'map'], + ['electronic resource', 'webpage'], + ['computer CD, DVD, USB', 'computerProgram'], + ['article, component part ', 'journalArticle'] + // there are likely other catalog item types in COBISS, + // which could be added to this hash later if they're being + // imported with the wrong Zotero item type + + ]); + return (catalogItemTypeHash.get(englishCatalogItemType)); +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrape(doc, url = doc.location.href) { + var finalItemType = ""; + // if url matches /en/bib/, then skip constructing englishURL + if (url.match("/en/bib")) { + // get catalog item type from page, then translate to Zotero item type using translateItemType() + var nativeEnglishItemType = doc.querySelector("button#add-biblioentry-to-shelf").getAttribute("data-mat-type"); + finalItemType = translateItemType(nativeEnglishItemType); + } + else { + // replace specific language in bib record URL with english to detect item type + var englishURL = constructEnglishURL(url); + var englishDocument = await requestDocument(englishURL); + var englishItemType = englishDocument.querySelector("button#add-biblioentry-to-shelf").getAttribute("data-mat-type"); + finalItemType = translateItemType(englishItemType); + } + if (doc.getElementById("unpaywall-link")) { + var pdfLink = doc.getElementById("unpaywall-link").href; + } + if (doc.getElementById('showUrlHref')) { + var fullTextLink = doc.getElementById('showUrlHref').href; + } + + const risURL = constructRISURL(url); + const risText = await requestText(risURL); + // case for catalog items with RIS (95%+ of items) + if (risText) { + // RIS always has an extraneous OK## at the beginning, remove it + let fixedRisText = risText.replace(/^OK##/, ''); + // PY tag sometimes has 'cop.' at the end - remove it or it makes the date parser return '0000' for some reason + fixedRisText = fixedRisText.replace(/^(PY\s*-\s*.+)cop\.$/m, '$1'); + const translator = Zotero.loadTranslator('import'); + translator.setTranslator('32d59d2d-b65a-4da4-b0a3-bdd3cfb979e7'); // RIS + translator.setString(fixedRisText); + translator.setHandler('itemDone', (_obj, item) => { + if (pdfLink) { + item.attachments.push({ + url: pdfLink, + title: 'Full Text PDF', + mimeType: 'application/pdf' + }); + } + else if (fullTextLink) { + if (fullTextLink.match(/.pdf$/)) { + item.attachments.push({ + url: fullTextLink, + title: 'Full Text PDF', + mimeType: 'application/pdf' + }); + } + else { + item.attachments.push({ + url: fullTextLink, + title: 'Full Text', + mimeType: 'text/html' + }); + } + } + + // if finalItemType is found from the catalog page, override itemType from RIS with it. + // if "Type of material" from catalog page isn't in catalogItemTypeHash, finalItemType will return as undefined. + // in this case, default Type from RIS will remain. + if (finalItemType) { + item.itemType = finalItemType; + } + + // some items have tags in RIS KW field and are captured by + // RIS translator, e.g. https://plus.cobiss.net/cobiss/si/en/bib/78691587. + // don't add dupliicate tags from the page to these items. + if (item.tags.length === 0) { + // other items e.g. https://plus.cobiss.net/cobiss/si/sl/bib/82789891 have tags, + // but they're not in the RIS. In this case, add tags from catalog page. + var pageTags = doc.querySelectorAll('a[href^="bib/search?c=su="]'); + for (let tagElem of pageTags) { + item.tags.push(tagElem.innerText); + } + } + item.url = url; + item.complete(); + }); + await translator.translate(); + } + + // case for catalog items with no RIS (remaining 5% or so of items) where we can't use the RIS import translator + else { + // construct correct fullRecord URL from basic catalog URL or #full URL + // base URL: https://plus.cobiss.net/cobiss/si/sl/bib/93266179 + // JSON URL: https://plus.cobiss.net/cobiss/si/sl/bib/COBIB/93266179/full + var jsonUrl = url.replace(/\/bib\/(\d+)/, "/bib/COBIB/$1/full"); + var fullRecord = await requestJSON(jsonUrl); + var noRISItem = new Zotero.Item(finalItemType); + noRISItem.title = fullRecord.titleCard.value; + var creatorsJson = fullRecord.author700701.value; + var brSlashRegex = //; + var creators = creatorsJson.split(brSlashRegex).map(value => value.trim()); + for (let creator of creators) { + // creator role isn't defined in metadata, so assign everyone "author" role + let role = "author"; + noRISItem.creators.push(ZU.cleanAuthor(creator, role, true)); + } + if (fullRecord.languageCard) noRISItem.language = fullRecord.languageCard.value; + if (fullRecord.publishDate) noRISItem.date = fullRecord.publishDate.value; + if (fullRecord.edition) noRISItem.edition = fullRecord.edition.value; + if (fullRecord.isbnCard) noRISItem.ISBN = fullRecord.isbnCard.value; + + if (fullRecord.publisherCard) { + var placePublisher = fullRecord.publisherCard.value; + // example string for publisherCard.value: "Ljubljana : Intelego, 2022" + const colonIndex = placePublisher.indexOf(":"); + const commaIndex = placePublisher.indexOf(","); + noRISItem.place = placePublisher.slice(0, colonIndex).trim(); + noRISItem.publisher = placePublisher.slice(colonIndex + 2, commaIndex).trim(); + } + + if (fullRecord.notesCard) { + var notesJson = fullRecord.notesCard.value; + var brRegex = /
    /; + var notes = notesJson.split(brRegex).map(value => value.trim()); + for (let note of notes) { + noRISItem.notes.push(note); + } + } + + // add subjects from JSON as tags. There are three fields that contain tags, + // sgcHeadings, otherSubjects and subjectCardUncon with + // different separators. sgcHeadings and otherSubjects use
    , subjectCardUncon uses / + if (fullRecord.sgcHeadings) { + var sgcHeadingsJson = fullRecord.sgcHeadings.value; + var sgcHeadingTags = sgcHeadingsJson.split(brRegex).map(value => value.trim()); + for (let sgcHeadingTag of sgcHeadingTags) { + noRISItem.tags.push(sgcHeadingTag); + } + } + + if (fullRecord.otherSubjects) { + var otherSubjectsJson = fullRecord.otherSubjects.value; + var otherSubjectsTags = otherSubjectsJson.split(brRegex).map(value => value.trim()); + for (let otherSubjectsTag of otherSubjectsTags) { + noRISItem.tags.push(otherSubjectsTag); + } + } + + if (fullRecord.subjectCardUncon) { + var subjectCardUnconJson = fullRecord.subjectCardUncon.value; + const slashRegex = /\//; + var subjectCardUnconTags = subjectCardUnconJson.split(slashRegex).map(value => value.trim()); + for (let subjectCardUnconTag of subjectCardUnconTags) { + noRISItem.tags.push(subjectCardUnconTag); + } + } + // add attachments to RIS items + if (pdfLink) { + noRISItem.attachments.push({ + url: pdfLink, + title: 'Full Text PDF', + mimeType: 'application/pdf' + }); + } + else if (fullTextLink) { + if (fullTextLink.match(/.pdf$/)) { + noRISItem.attachments.push({ + url: fullTextLink, + title: 'Full Text PDF', + mimeType: 'application/pdf' + }); + } + else { + noRISItem.attachments.push({ + url: fullTextLink, + title: 'Full Text', + mimeType: 'text/html' + }); + } + } + noRISItem.complete(); + } +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/92020483", + "items": [ + { + "itemType": "videoRecording", + "title": "Nauk o barvah po Goetheju. DVD 2/3, Poglobitev vsebine nauka o barvah, še posebej poglavja \"Fizične barve\" s prikazom eksperimentov", + "creators": [ + { + "lastName": "Kühl", + "firstName": "Johannes", + "creatorType": "director" + } + ], + "date": "2022", + "ISBN": "9789619527542", + "libraryCatalog": "COBISS", + "place": "Hvaletinci", + "studio": "NID Sapientia", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/92020483", + "attachments": [], + "tags": [ + { + "tag": "Antropozofija" + }, + { + "tag": "Barve" + } + ], + "notes": [ + { + "note": "

    Dialogi v slov. in nem. s konsekutivnim prevodom v slov.

    " + }, + { + "note": "

    Tisk po naročilu

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/search?q=*&db=cobib&mat=allmaterials&cof=0_105b-p&pdfrom=01.01.2023", + "items": "multiple" + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/115256576", + "items": [ + { + "itemType": "book", + "title": "Angel z zahodnega okna", + "creators": [ + { + "lastName": "Meyrink", + "firstName": "Gustav", + "creatorType": "author" + } + ], + "date": "2001", + "ISBN": "9789616400107", + "libraryCatalog": "COBISS", + "numPages": "2 zv. (216; 203 )", + "place": "Ljubljana", + "publisher": "Založniški atelje Blodnjak", + "series": "Zbirka Blodnjak", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/115256576", + "attachments": [], + "tags": [], + "notes": [ + { + "note": "

    Prevod dela: Der Engel vom westlichen Fenster

    " + }, + { + "note": "

    Gustav Meyrink / Herman Hesse: str. 198-200

    " + }, + { + "note": "

    Magični stekleni vrtovi judovske kulture / Jorge Luis Borges: str. 201-203

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/139084803", + "detectedItemType": "book", + "items": [ + { + "itemType": "report", + "title": "Poročilo analiz vzorcev odpadnih vod na vsebnost prepovedanih in dovoljenih drog na področju centralne čistilne naprave Kranj (2022)", + "creators": [ + { + "lastName": "Heath", + "firstName": "Ester", + "creatorType": "author" + }, + { + "lastName": "Verovšek", + "firstName": "Taja", + "creatorType": "author" + } + ], + "date": "2023", + "institution": "Institut Jožef Stefan", + "libraryCatalog": "COBISS", + "pages": "1 USB-ključ", + "place": "Ljubljana", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/139084803", + "attachments": [], + "tags": [ + { + "tag": "dovoljene droge" + }, + { + "tag": "nedovoljene droge" + }, + { + "tag": "odpadne vode" + }, + { + "tag": "čistilna naprava" + } + ], + "notes": [ + { + "note": "

    Nasl. z nasl. zaslona

    " + }, + { + "note": "

    Opis vira z dne 11. 1. 2023

    " + }, + { + "note": "

    Bibliografija: str. 13

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/84534787", + "detectedItemType": "book", + "items": [ + { + "itemType": "journalArticle", + "title": "Flood legislation and land policy framework of EU and non-EU countries in Southern Europe", + "creators": [ + { + "lastName": "Kapović-Solomun", + "firstName": "Marijana", + "creatorType": "author" + }, + { + "lastName": "Ferreira", + "firstName": "Carla S.S.", + "creatorType": "author" + }, + { + "lastName": "Zupanc", + "firstName": "Vesna", + "creatorType": "author" + }, + { + "lastName": "Ristić", + "firstName": "Ratko", + "creatorType": "author" + }, + { + "lastName": "Drobnjak", + "firstName": "Aleksandar", + "creatorType": "author" + }, + { + "lastName": "Kalantari", + "firstName": "Zahra", + "creatorType": "author" + } + ], + "date": "2022", + "ISSN": "2049-1948", + "issue": "1", + "journalAbbreviation": "WIREs", + "libraryCatalog": "COBISS", + "pages": "1-14", + "publicationTitle": "WIREs", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/84534787", + "volume": "9", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "EU legislation" + }, + { + "tag": "Južna Evropa" + }, + { + "tag": "Southern Europe" + }, + { + "tag": "floods" + }, + { + "tag": "land governance" + }, + { + "tag": "policy framework" + }, + { + "tag": "politika" + }, + { + "tag": "poplave" + }, + { + "tag": "upravljanje zemljišč" + }, + { + "tag": "zakonodaja EU" + } + ], + "notes": [ + { + "note": "

    Nasl. z nasl. zaslona

    " + }, + { + "note": "

    Opis vira z dne 11. 11. 2021

    " + }, + { + "note": "

    Bibliografija: str. 12-14

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/5815649", + "detectedItemType": "book", + "items": [ + { + "itemType": "thesis", + "title": "Rangiranje cest po metodologiji EuroRAP ; Elektronski vir: diplomska naloga = Rating roads using EuroRAP procedures", + "creators": [ + { + "lastName": "Pešec", + "firstName": "Katja", + "creatorType": "author" + } + ], + "date": "2012", + "libraryCatalog": "COBISS", + "place": "Ljubljana", + "shortTitle": "Rangiranje cest po metodologiji EuroRAP ; Elektronski vir", + "university": "[K. Pešec]", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/5815649", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "EuroRAP" + }, + { + "tag": "EuroRAP" + }, + { + "tag": "VSŠ" + }, + { + "tag": "cesta in obcestje" + }, + { + "tag": "diplomska dela" + }, + { + "tag": "economic efficiency" + }, + { + "tag": "ekonomska učinkovitost" + }, + { + "tag": "gradbeništvo" + }, + { + "tag": "graduation thesis" + }, + { + "tag": "pilot project" + }, + { + "tag": "pilotski projekt" + }, + { + "tag": "predlagani (proti)ukrepi" + }, + { + "tag": "rangiranje cest" + }, + { + "tag": "road and roadside" + }, + { + "tag": "star rating" + }, + { + "tag": "suggested countermeasure" + } + ], + "notes": [ + { + "note": "

    Diplomsko delo visokošolskega strokovnega študija gradbeništva, Prometna smer

    " + }, + { + "note": "

    Nasl. z nasl. zaslona

    " + }, + { + "note": "

    Publikacija v pdf formatu obsega 103 str.

    " + }, + { + "note": "

    Bibliografija: str. 85-87

    " + }, + { + "note": "

    Izvleček ; Abstract

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/82789891", + "detectedItemType": "book", + "items": [ + { + "itemType": "conferencePaper", + "title": "Posvet Avtomatizacija strege in montaže 2021/2021 - ASM '21/22, Ljubljana, 11. 05. 2022: zbornik povzetkov s posveta", + "creators": [ + { + "lastName": "Posvet Avtomatizacija strege in montaže", + "creatorType": "author", + "fieldMode": 1 + }, + { + "lastName": "Herakovič", + "firstName": "Niko", + "creatorType": "editor" + }, + { + "lastName": "Debevec", + "firstName": "Mihael", + "creatorType": "editor" + }, + { + "lastName": "Pipan", + "firstName": "Miha", + "creatorType": "editor" + }, + { + "lastName": "Adrović", + "firstName": "Edo", + "creatorType": "editor" + } + ], + "date": "2022", + "ISBN": "9789616980821", + "libraryCatalog": "COBISS", + "pages": "141", + "place": "Ljubljana", + "publisher": "Fakulteta za strojništvo", + "shortTitle": "Posvet Avtomatizacija strege in montaže 2021/2021 - ASM '21/22, Ljubljana, 11. 05. 2022", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/82789891", + "attachments": [], + "tags": [ + { + "tag": "Avtomatizacija" + }, + { + "tag": "Posvetovanja" + }, + { + "tag": "Strojništvo" + } + ], + "notes": [ + { + "note": "

    180 izv.

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/78691587", + "items": [ + { + "itemType": "thesis", + "title": "Modeliranje obratovanja transformatorskih postaj z metodami strojnega učenja: diplomsko delo: visokošolski strokovni študijski program prve stopnje Računalništvo in informatika", + "creators": [ + { + "lastName": "Čuš", + "firstName": "Tibor", + "creatorType": "author" + } + ], + "date": "2022", + "libraryCatalog": "COBISS", + "numPages": "55", + "place": "Ljubljana", + "shortTitle": "Modeliranje obratovanja transformatorskih postaj z metodami strojnega učenja", + "university": "[T. Čuš]", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/78691587", + "attachments": [ + { + "title": "Full Text", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "computer science" + }, + { + "tag": "diploma" + }, + { + "tag": "diplomske naloge" + }, + { + "tag": "electrical power system" + }, + { + "tag": "elektroenergetski sistem" + }, + { + "tag": "forecasting models" + }, + { + "tag": "indikatorji preobremenitev" + }, + { + "tag": "machine learning" + }, + { + "tag": "napovedni modeli" + }, + { + "tag": "overload indicators" + }, + { + "tag": "transformer station" + }, + { + "tag": "visokošolski strokovni študij" + } + ], + "notes": [ + { + "note": "

    Bibliografija: str. 53-55

    " + }, + { + "note": "

    Povzetek ; Abstract: Modeling transformer station operation with machine learning methods

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/94705155#full", + "items": [ + { + "itemType": "book", + "title": "Ljubezen v pismih: dopisovanje med Felicito Koglot in Francem Pericem: Aleksandrija-Bilje: 1921-1931", + "creators": [ + { + "lastName": "Koglot", + "firstName": "Felicita", + "creatorType": "author" + }, + { + "lastName": "Peric", + "firstName": "Franc", + "creatorType": "author" + }, + { + "lastName": "Vončina", + "firstName": "Lara", + "creatorType": "editor" + }, + { + "lastName": "Orel", + "firstName": "Maja", + "creatorType": "editor" + }, + { + "lastName": "Koren", + "firstName": "Manca", + "creatorType": "editor" + }, + { + "lastName": "Mihurko Poniž", + "firstName": "Katja", + "creatorType": "editor" + } + ], + "date": "2022", + "ISBN": "9789617025224", + "libraryCatalog": "COBISS", + "numPages": "235", + "place": "V Novi Gorici", + "publisher": "Založba Univerze", + "shortTitle": "Ljubezen v pismih", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/94705155#full", + "attachments": [], + "tags": [ + { + "tag": "Primorska" + }, + { + "tag": "Slovenke" + }, + { + "tag": "emigracija" + }, + { + "tag": "pisma" + }, + { + "tag": "ženske" + } + ], + "notes": [ + { + "note": "

    Potiskane notr. str. ov.

    " + }, + { + "note": "

    250 izv.

    " + }, + { + "note": "

    Kdo sta bila Felicita Koglot in Franc Peric in o knjižni izdaji njunega dopisovanja / Manca Koren, Maja Orel, Lara Vončina: str. 5-6

    " + }, + { + "note": "

    Kratek oris zgodovinskih razmer v Egiptu in na Primorskem v obdobju med obema vojnama / Manca Koren: str. 185-195

    " + }, + { + "note": "

    Franc Peric in Felicita Koglot: večkratne migracije v družinski korespondenci / Mirjam Milharčič Hladnik: str. 197-209

    " + }, + { + "note": "

    Družinsko življenje in doživljanje aleksandrinstva v pismih / Manca Koren: str. 211-228

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/78691587", + "items": [ + { + "itemType": "thesis", + "title": "Modeliranje obratovanja transformatorskih postaj z metodami strojnega učenja: diplomsko delo: visokošolski strokovni študijski program prve stopnje Računalništvo in informatika", + "creators": [ + { + "lastName": "Čuš", + "firstName": "Tibor", + "creatorType": "author" + } + ], + "date": "2022", + "libraryCatalog": "COBISS", + "numPages": "55", + "place": "Ljubljana", + "shortTitle": "Modeliranje obratovanja transformatorskih postaj z metodami strojnega učenja", + "university": "[T. Čuš]", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/78691587", + "attachments": [ + { + "title": "Full Text", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "computer science" + }, + { + "tag": "diploma" + }, + { + "tag": "diplomske naloge" + }, + { + "tag": "electrical power system" + }, + { + "tag": "elektroenergetski sistem" + }, + { + "tag": "forecasting models" + }, + { + "tag": "indikatorji preobremenitev" + }, + { + "tag": "machine learning" + }, + { + "tag": "napovedni modeli" + }, + { + "tag": "overload indicators" + }, + { + "tag": "transformer station" + }, + { + "tag": "visokošolski strokovni študij" + } + ], + "notes": [ + { + "note": "

    Bibliografija: str. 53-55

    " + }, + { + "note": "

    Povzetek ; Abstract: Modeling transformer station operation with machine learning methods

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/search?q=*&db=cobib&mat=allmaterials&cof=0_105b-mb16", + "items": "multiple" + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/101208835", + "items": [ + { + "itemType": "book", + "title": "Fizika. Zbirka maturitetnih nalog z rešitvami 2012-2017 / [avtorji Vitomir Babič ... [et al.] ; urednika Aleš Drolc, Joži Trkov]", + "creators": [ + { + "firstName": "Vito", + "lastName": "Babič", + "creatorType": "author" + }, + { + "firstName": "Ruben", + "lastName": "Belina", + "creatorType": "author" + }, + { + "firstName": "Peter", + "lastName": "Gabrovec", + "creatorType": "author" + }, + { + "firstName": "Marko", + "lastName": "Jagodič", + "creatorType": "author" + }, + { + "firstName": "Aleš", + "lastName": "Mohorič", + "creatorType": "author" + }, + { + "firstName": "Mirijam", + "lastName": "Pirc", + "creatorType": "author" + }, + { + "firstName": "Gorazd", + "lastName": "Planinšič", + "creatorType": "author" + }, + { + "firstName": "Mitja", + "lastName": "Slavinec", + "creatorType": "author" + }, + { + "firstName": "Ivica", + "lastName": "Tomić", + "creatorType": "author" + } + ], + "date": "2022", + "ISBN": "9789616899420", + "edition": "3. ponatis", + "language": "Slovenian", + "libraryCatalog": "COBISS", + "place": "Ljubljana", + "publisher": "Državni izpitni center", + "attachments": [], + "tags": [ + { + "tag": "Fizika -- Matura -- 2012-2017 -- Vaje za srednje šole" + }, + { + "tag": "Fizika -- Vaje za maturo" + }, + { + "tag": "izpitne naloge za srednje šole" + }, + { + "tag": "naloge" + }, + { + "tag": "rešitve" + }, + { + "tag": "testi znanja" + }, + { + "tag": "učbeniki za srednje šole" + } + ], + "notes": [ + "Nasl. na hrbtu: Fizika 2012-2017", + "Avtorji navedeni v kolofonu", + "600 izv." + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/sl/bib/93266179", + "items": [ + { + "itemType": "book", + "title": "Matematika na splošni maturi : 2022 : vprašanja in odgovori za ustni izpit iz matematike na splošni maturi za osnovno raven / Bojana Dvoržak", + "creators": [ + { + "firstName": "Bojana", + "lastName": "Dvoržak", + "creatorType": "author" + } + ], + "date": "2022", + "ISBN": "9789616558624", + "edition": "1. izd.", + "language": "slovenski", + "libraryCatalog": "COBISS", + "place": "Ljubljana", + "publisher": "Intelego", + "shortTitle": "Matematika na splošni maturi", + "attachments": [], + "tags": [ + { + "tag": "Matematika" + }, + { + "tag": "Matematika -- Katalogi znanja za srednje šole" + }, + { + "tag": "Matematika -- Matura -- Vaje za srednje šole" + }, + { + "tag": "Matematika -- Vaje za maturo" + }, + { + "tag": "Matura" + }, + { + "tag": "Naloge, vaje itd." + }, + { + "tag": "izpitne naloge za srednje šole" + }, + { + "tag": "odgovori" + }, + { + "tag": "osnovna raven" + }, + { + "tag": "rešitve" + }, + { + "tag": "testi znanja" + }, + { + "tag": "učbeniki za srednje šole" + }, + { + "tag": "vprašanja" + }, + { + "tag": "zaključni izpiti" + } + ], + "notes": [ + "Dodatek k nasl. v kolofonu in CIP-u: Vprašanja in odgovori za ustni izpit iz matematike na splošni maturi 2022 za osnovno raven", + "1.000 izv." + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/143385859", + "items": [ + { + "itemType": "book", + "title": "Fizika. Zbirka maturitetnih nalog z rešitvami 2012-2017 / [avtorji Vitomir Babič ... [et al.] ; urednika Aleš Drolc, Joži Trkov]", + "creators": [ + { + "firstName": "Vito", + "lastName": "Babič", + "creatorType": "author" + }, + { + "firstName": "Ruben", + "lastName": "Belina", + "creatorType": "author" + }, + { + "firstName": "Peter", + "lastName": "Gabrovec", + "creatorType": "author" + }, + { + "firstName": "Marko", + "lastName": "Jagodič", + "creatorType": "author" + }, + { + "firstName": "Aleš", + "lastName": "Mohorič", + "creatorType": "author" + }, + { + "firstName": "Mirijam", + "lastName": "Pirc", + "creatorType": "author" + }, + { + "firstName": "Gorazd", + "lastName": "Planinšič", + "creatorType": "author" + }, + { + "firstName": "Mitja", + "lastName": "Slavinec", + "creatorType": "author" + }, + { + "firstName": "Ivica", + "lastName": "Tomić", + "creatorType": "author" + } + ], + "date": "2023", + "ISBN": "9789616899420", + "edition": "4. ponatis", + "language": "Slovenian", + "libraryCatalog": "COBISS", + "place": "Ljubljana", + "publisher": "Državni izpitni center", + "attachments": [], + "tags": [ + { + "tag": "Fizika -- Matura -- 2012-2017 -- Priročniki" + }, + { + "tag": "Fizika -- Vaje za maturo" + }, + { + "tag": "izpitne naloge za srednje šole" + }, + { + "tag": "naloge" + }, + { + "tag": "rešitve" + }, + { + "tag": "učbeniki za srednje šole" + }, + { + "tag": "vaje za srednje šole" + } + ], + "notes": [ + "Hrbtni nasl.: Fizika 2012-2017", + "Avtorji navedeni v kolofonu", + "300 izv." + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/70461955", + "items": [ + { + "itemType": "book", + "title": "Napredna znanja za kakovostno mentorstvo v zdravstveni negi: znanstvena monografija", + "creators": [ + { + "lastName": "Filej", + "firstName": "Bojana", + "creatorType": "editor" + }, + { + "lastName": "Kaučič", + "firstName": "Boris Miha", + "creatorType": "editor" + } + ], + "date": "2023", + "ISBN": "9789616889377", + "edition": "1. izd.", + "libraryCatalog": "COBISS", + "place": "Celje", + "publisher": "Fakulteta za zdravstvene vede", + "shortTitle": "Napredna znanja za kakovostno mentorstvo v zdravstveni negi", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/70461955", + "attachments": [ + { + "title": "Full Text", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Izobrževanje" + }, + { + "tag": "Mentorstvo" + }, + { + "tag": "Mentorstvo" + }, + { + "tag": "Praktična znanja" + }, + { + "tag": "Vzgoja in izobraževanje" + }, + { + "tag": "Zborniki" + }, + { + "tag": "Zdravstvena nega" + }, + { + "tag": "Zdravstvena nega" + } + ], + "notes": [ + { + "note": "

    Nasl. z nasl. zaslona

    " + }, + { + "note": "

    Dokument v pdf formatu obsega 94 str.

    " + }, + { + "note": "

    Opis vira z dne 1. 2. 2023

    " + }, + { + "note": "

    Bibliografija pri posameznih poglavjih

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/105123075", + "items": [ + { + "itemType": "report", + "title": "Storitveni sektor in siva ekonomija v času epidemije COVID-19: raziskovalno delo: področje: ekonomija in turizem", + "creators": [ + { + "lastName": "Hochkraut", + "firstName": "Nataša", + "creatorType": "author" + }, + { + "lastName": "Verbovšek", + "firstName": "Lea", + "creatorType": "author" + } + ], + "date": "2022", + "institution": "Osnovna šola Primoža Trubarja", + "libraryCatalog": "COBISS", + "place": "Laško", + "shortTitle": "Storitveni sektor in siva ekonomija v času epidemije COVID-19", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/105123075", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "COVID 19" + }, + { + "tag": "SARS-Cov-2" + }, + { + "tag": "izvajalci storitev" + }, + { + "tag": "korelacija" + }, + { + "tag": "koronavirus" + }, + { + "tag": "potrošniki" + }, + { + "tag": "raziskovalne naloge" + }, + { + "tag": "siva ekonomija" + }, + { + "tag": "statistika" + }, + { + "tag": "storitveni sektor" + } + ], + "notes": [ + { + "note": "

    Raziskovalna naloga v okviru projekta Mladi za Celje 2022

    " + }, + { + "note": "

    Povzetek v slov in angl.

    " + }, + { + "note": "

    Bibliografija: f. 35-36

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/84576259", + "items": [ + { + "itemType": "audioRecording", + "title": "Reforma: tribute to Laibach", + "creators": [ + { + "lastName": "Noctiferia", + "creatorType": "composer", + "fieldMode": 1 + } + ], + "date": "2021", + "label": "Nika", + "libraryCatalog": "COBISS", + "place": "Ljubljana", + "shortTitle": "Reforma", + "url": "https://plus.cobiss.net/cobiss/si/en/bib/84576259", + "attachments": [ + { + "title": "Full Text", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "avantgardna glasba" + }, + { + "tag": "avantgardni rock" + }, + { + "tag": "black metal" + }, + { + "tag": "death metal" + }, + { + "tag": "extreme metal" + }, + { + "tag": "heavy metal" + }, + { + "tag": "industrial metal" + }, + { + "tag": "metal" + }, + { + "tag": "priredbe" + } + ], + "notes": [ + { + "note": "

    Leto posnetja 2021

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/al/sq/bib/334906368", + "items": [ + { + "itemType": "book", + "title": "Trëndafili i mesnatës", + "creators": [ + { + "lastName": "Riley", + "firstName": "Lucinda", + "creatorType": "author" + } + ], + "date": "[2022]", + "ISBN": "9789928366108", + "libraryCatalog": "COBISS", + "numPages": "603 f.", + "place": "[Tiranë]", + "publisher": "Dituria", + "series": "Letërsi e huaj bashkëkohore", + "url": "https://plus.cobiss.net/cobiss/al/sq/bib/334906368", + "attachments": [], + "tags": [ + { + "tag": "letërsia irlandeze" + }, + { + "tag": "romane" + } + ], + "notes": [ + { + "note": "

    Tit. i origj.: The midnight rose

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/bh/sr/bib/47388678", + "detectedItemType": "book", + "items": [ + { + "itemType": "thesis", + "title": "The influence of negative transfer on the use of collocations in high school student's writing = Utjecaj negativnog transfera na korištenje kolokacija u pismenim zadaćama učenika srednjih škola", + "creators": [ + { + "lastName": "Đapo", + "firstName": "Amra", + "creatorType": "author" + } + ], + "date": "2022", + "libraryCatalog": "COBISS", + "numPages": "76 listova", + "place": "Tuzla", + "university": "[A. Đapo]", + "url": "https://plus.cobiss.net/cobiss/bh/sr/bib/47388678", + "attachments": [], + "tags": [ + { + "tag": "acquisition" + }, + { + "tag": "collocations" + }, + { + "tag": "engleski kao drugi jezik" + }, + { + "tag": "errors" + }, + { + "tag": "greške" + }, + { + "tag": "kolokacije" + }, + { + "tag": "magistarski rad" + }, + { + "tag": "transfer" + }, + { + "tag": "transfer" + }, + { + "tag": "usvajanje" + } + ], + "notes": [ + { + "note": "

    Bibliografija: listovi 73-76

    " + }, + { + "note": "

    Sažetak ; Summary

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/rs/sr/bib/57790729", + "detectedItemType": "book", + "items": [ + { + "itemType": "statute", + "nameOfAct": "Закон о Централном регистру обавезног социјалног осигурања, са подзаконским актима", + "creators": [ + { + "lastName": "Србија", + "creatorType": "author", + "fieldMode": 1 + }, + { + "lastName": "Мартић", + "firstName": "Вера", + "creatorType": "editor" + } + ], + "dateEnacted": "2022", + "pages": "II, 74 стр.", + "url": "https://plus.cobiss.net/cobiss/rs/sr/bib/57790729", + "attachments": [], + "tags": [ + { + "tag": "Београд" + }, + { + "tag": "Законски прописи" + }, + { + "tag": "Централни регистар обавезног социјалног осигурања" + } + ], + "notes": [ + { + "note": "

    Тираж 300

    " + }, + { + "note": "

    Напомене и библиографске референце уз текст.

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/bg/en/bib/51193608", + "items": [ + { + "itemType": "thesis", + "title": "Хирургични аспекти на аноректалните абсцеси при деца и възрастни: дисертационен труд за присъждане на образователна и научна степен \"доктор\", област на висше образование 7. Здравеопазване и спорт, професионално направление 7.1 Медицина, научна специалност: 03.01.37 Обща хирургия", + "creators": [ + { + "lastName": "Хаджиева", + "firstName": "Елена Божидарова", + "creatorType": "author" + }, + { + "lastName": "Hadžieva", + "firstName": "Elena Božidarova", + "creatorType": "author" + } + ], + "date": "2022", + "libraryCatalog": "COBISS", + "numPages": "198 л.", + "place": "Пловдив", + "shortTitle": "Хирургични аспекти на аноректалните абсцеси при деца и възрастни", + "university": "[Е. Хаджиева]", + "url": "https://plus.cobiss.net/cobiss/bg/en/bib/51193608", + "attachments": [], + "tags": [], + "notes": [ + { + "note": "

    Библиогр.: л. 175-190

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/ks/sq/bib/120263427", + "detectedItemType": "book", + "items": [ + { + "itemType": "conferencePaper", + "title": "Kumtesat nga konferenca shkencore ndërkombëtare: (17 dhe 18 nëntor 2021): Ndikimi i COVID-19 në humbjet mësimore - pasojat në rritjen e pabarazive në mësim dhe sfidat e përmbushjes/kompensimit", + "creators": [ + { + "lastName": "Instituti Pedagogjik i Kosovës", + "firstName": "Konferenca shkencore ndërkombëtare", + "creatorType": "author" + }, + { + "lastName": "Koliqi", + "firstName": "Hajrullah", + "creatorType": "editor" + } + ], + "date": "2021", + "ISBN": "9789951591560", + "libraryCatalog": "COBISS", + "pages": "190 f.", + "place": "Prishtinë", + "publisher": "Instituti Pedagogjik i Kosovës", + "shortTitle": "Kumtesat nga konferenca shkencore ndërkombëtare", + "url": "https://plus.cobiss.net/cobiss/ks/sq/bib/120263427", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "covid-19" + }, + { + "tag": "mësimi online" + }, + { + "tag": "përmbledhjet e punimeve" + }, + { + "tag": "sistemet arsimore" + } + ], + "notes": [ + { + "note": "

    Përmbledhjet në gjuhën shqipe dhe angleze

    " + }, + { + "note": "

    Bibliografia në fund të çdo punimi

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/mk/en/bib/search?q=*&db=cobib&mat=allmaterials&tyf=1_gla_cd", + "items": "multiple" + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/mk/mk/bib/57036037", + "detectedItemType": "book", + "items": [ + { + "itemType": "audioRecording", + "title": "Крени ме", + "creators": [ + { + "lastName": "Кајшаров", + "firstName": "Константин", + "creatorType": "composer" + } + ], + "date": "2022", + "label": "К. Кајшаров", + "libraryCatalog": "COBISS", + "place": "Скопје", + "url": "https://plus.cobiss.net/cobiss/mk/mk/bib/57036037", + "attachments": [], + "tags": [ + { + "tag": "CD-a" + }, + { + "tag": "Вокално-инструментални композиции" + }, + { + "tag": "Духовна музика" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/cg/cnr_cyrl/bib/20926212", + "detectedItemType": "book", + "items": [ + { + "itemType": "thesis", + "title": "Menadžment ljudskih resursa: diplomski rad", + "creators": [ + { + "lastName": "Obradović", + "firstName": "Nikoleta", + "creatorType": "author" + } + ], + "date": "2022", + "libraryCatalog": "COBISS", + "place": "Podgorica", + "shortTitle": "Menadžment ljudskih resursa", + "university": "[N. Obradović]", + "url": "https://plus.cobiss.net/cobiss/cg/cnr_cyrl/bib/20926212", + "attachments": [], + "tags": [ + { + "tag": "Diplomski radovi" + }, + { + "tag": "Menadžment ljudskih resursa" + } + ], + "notes": [ + { + "note": "

    Nasl. sa nasl. ekrana

    " + }, + { + "note": "

    Bibliografija

    " + } + ], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/sr/sr_latn/bib/search?q=*&db=cobib&mat=allmaterials", + "items": "multiple" + }, + { + "type": "web", + "url": "https://plus.cobiss.net/cobiss/sr/sr_latn/bib/15826441", + "items": [ + { + "itemType": "book", + "title": "Zanosni", + "creators": [ + { + "lastName": "Prince", + "creatorType": "author", + "fieldMode": 1 + }, + { + "lastName": "Принс", + "creatorType": "author", + "fieldMode": 1 + }, + { + "lastName": "Božić", + "firstName": "Aleksandar", + "creatorType": "editor" + }, + { + "lastName": "Божић", + "firstName": "Александар", + "creatorType": "editor" + } + ], + "date": "2022", + "ISBN": "9788664630160", + "libraryCatalog": "COBISS", + "numPages": "281", + "place": "Beograd", + "publisher": "IPC Media", + "series": "Edicija (B)io", + "url": "https://plus.cobiss.net/cobiss/sr/sr_latn/bib/15826441", + "attachments": [], + "tags": [ + { + "tag": "Аутобиографија" + }, + { + "tag": "Принс, 1958-2016" + } + ], + "notes": [ + { + "note": "

    Prevod dela: The beautiful ones / Prince

    " + }, + { + "note": "

    Autorove slike

    " + }, + { + "note": "

    Tiraž 1.000

    " + }, + { + "note": "

    Str. 4-49: Predgovor / Den Pajpenbring

    " + }, + { + "note": "

    O autorima: str. [282].

    " + } + ], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/Cairn.info.js b/Cairn.info.js index 19aa2510b48..411382cc93a 100644 --- a/Cairn.info.js +++ b/Cairn.info.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-05-31 18:42:02" + "lastUpdated": "2023-10-23 08:08:57" } /* @@ -80,11 +80,9 @@ function getSearchResults(doc, checkOnly) { async function doWeb(doc, url) { if (await detectWeb(doc, url) == 'multiple') { let items = await Zotero.selectItems(getSearchResults(doc, false)); - if (items) { - await Promise.all( - Object.keys(items) - .map(url => requestDocument(url).then(scrape)) - ); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } } else { @@ -125,7 +123,7 @@ async function scrape(doc) { if (pdfLink) { item.attachments.push({ - url: pdfLink.href, + url: pdfLink, title: 'Full Text PDF', mimeType: 'application/pdf' }); diff --git a/Camara Brasileira do Livro ISBN.js b/Camara Brasileira do Livro ISBN.js new file mode 100644 index 00000000000..be20df316f8 --- /dev/null +++ b/Camara Brasileira do Livro ISBN.js @@ -0,0 +1,842 @@ +{ + "translatorID": "cdb5c893-ab69-4e96-9b5c-f4456d49ddd8", + "label": "Câmara Brasileira do Livro ISBN", + "creator": "Abe Jellinek", + "target": "", + "minVersion": "5.0", + "maxVersion": "", + "priority": 98, + "inRepository": true, + "translatorType": 8, + "lastUpdated": "2023-09-26 16:11:18" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Abe Jellinek + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +function detectSearch(items) { + items = cleanData(items); + return !!items.length; +} + +async function doSearch(items) { + items = cleanData(items); + for (let { ISBN } of items) { + let search = ISBN; + if (ISBN.length == 10) { + search += ' OR ' + ZU.toISBN13(ISBN); + } + let body = { + count: true, + facets: [], + filter: '', + orderby: null, + queryType: 'full', + search, + searchFields: 'FormattedKey,RowKey', + searchMode: 'any', + select: '*', + skip: 0, + top: 1 + }; + let response = await requestJSON('https://isbn-search-br.search.windows.net/indexes/isbn-index/docs/search?api-version=2016-09-01', { + method: 'POST', + headers: { + 'Content-Type': 'application/json; charset=UTF-8', + 'api-key': '100216A23C5AEE390338BBD19EA86D29', + Origin: 'https://www.cblservicos.org.br', + Referer: 'https://www.cblservicos.org.br/' + }, + body: JSON.stringify(body) + }); + let results = response.value; + for (let result of results) { + translateResult(result); + } + } +} + +function translateResult(result) { + Z.debug(result) + let item = new Zotero.Item('book'); + item.title = result.Title; + if (result.Subtitle && !item.title.includes(':') && !result.Subtitle.includes(':')) { + item.title += ': ' + result.Subtitle; + } + item.title = fixCase(item.title); + item.abstractNote = result.Sinopse; + item.series = fixCase(result.Colection); + // TODO: Need example data for: + // item.seriesNumber + // item.volume + // item.numberOfVolumes + item.edition = result.Edicao; + if (item.edition == '1') { + item.edition = ''; + } + item.place = (result.Cidade || '') + (result.UF ? ', ' + result.UF : ''); + item.publisher = fixCase(result.Imprint); + item.date = ZU.strToISO(result.Date); + item.numPages = result.Paginas; + if (item.numPages == '0') { + item.numPages = ''; + } + item.language = (result.IdiomasObra && result.IdiomasObra[0]) || 'pt-BR'; + if (item.language == 'português (Brasil)') { + item.language = 'pt-BR'; + } + item.ISBN = ZU.cleanISBN(result.FormattedKey); + for (let [i, author] of result.Authors.entries()) { + if (author == author.toUpperCase()) { + author = ZU.capitalizeName(author); + } + let creatorType; + if (result.Profissoes && result.Profissoes.length === result.Authors.length) { + switch (result.Profissoes[i]) { + case 'Coordenador': + case 'Autor': + case 'Roteirista': + creatorType = 'author'; + break; + case 'Revisor': + case 'Organizador': + case 'Editor': + creatorType = 'editor'; + break; + case 'Tradutor': + creatorType = 'translator'; + break; + case 'Ilustrador': // TODO: Used? + case 'Projeto Gráfico': + creatorType = 'illustrator'; + break; + default: + // First creator is probably an author, + // even if the Profissoes string is something weird + creatorType = i == 0 ? 'author' : 'contributor'; + break; + } + } + // No/mismatched-length Profissoes array, so we have to guess that this non-primary creator + // is a contributor + else if (i > 0) { + creatorType = 'contributor'; + } + // No/mismatched-length Profissoes array, so we have to guess that this primary creator + // is an author + else { + creatorType = 'author'; + } + // Brazilian names often contain many surnames, but determining which names are surnames + // and which are given names is outside the scope of this translator. + // Chicago indexes by the final element of the name alone, and so will we: + // https://en.wikipedia.org/wiki/Portuguese_name#Indexing + let creator = ZU.cleanAuthor(author, creatorType, author.includes(',')); + if (!creator.firstName) creator.fieldMode = 1; + + // That said, we will handle name suffixes, which should be combined with the last "middle" + // name particle in the last name + if (creator.firstName && creator.lastName + && ['filho', 'junior', 'neto', 'sobrinho', 'segundo', 'terceiro'] + .includes(ZU.removeDiacritics(creator.lastName.toLowerCase()))) { + let firstNameSplit = creator.firstName.split(/\s+/); + if (firstNameSplit.length) { + let lastParticleFirstName = firstNameSplit[firstNameSplit.length - 1]; + creator.lastName = lastParticleFirstName + ' ' + creator.lastName; + creator.firstName = firstNameSplit.slice(0, firstNameSplit.length - 1).join(' '); + } + } + item.creators.push(creator); + } + if (result.Subject) { + item.tags.push({ tag: result.Subject }); + } + for (let tag of result.PalavrasChave) { + item.tags.push({ tag }); + } + item.complete(); +} + +function fixCase(s) { + if (s && s == s.toUpperCase()) { + s = ZU.capitalizeTitle(s, true); + } + return s; +} + +function cleanData(items) { + if (!Array.isArray(items)) { + items = [items]; + } + return items + .map((item) => { + if (typeof item === 'string') { + item = { ISBN: item }; + } + if (item.ISBN) { + item.ISBN = ZU.cleanISBN(item.ISBN); + } + return item; + }) + .filter(item => item.ISBN && ( + item.ISBN.startsWith('97865') + || item.ISBN.startsWith('65') + || item.ISBN.startsWith('97885') + || item.ISBN.startsWith('85'))); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "search", + "input": { + "ISBN": "9786599594755" + }, + "items": [ + { + "itemType": "book", + "title": "Mercadores da Insegurança: conjuntura e riscos do hacking governamental no Brasil", + "creators": [ + { + "firstName": "André", + "lastName": "Ramiro", + "creatorType": "author" + }, + { + "firstName": "Pedro", + "lastName": "Amaral", + "creatorType": "contributor" + }, + { + "firstName": "Mariana", + "lastName": "Canto", + "creatorType": "contributor" + }, + { + "firstName": "Marcos César M.", + "lastName": "Pereira", + "creatorType": "contributor" + }, + { + "firstName": "Raquel", + "lastName": "Saraiva", + "creatorType": "contributor" + }, + { + "firstName": "Clara", + "lastName": "Guimarães", + "creatorType": "contributor" + } + ], + "date": "2022-10-11", + "ISBN": "9786599594755", + "abstractNote": "Em políticas públicas, o debate sobre como as técnicas de investigações criminais devem responder à digitalização das dinâmicas sociais tem se sobressaído e caminha em uma linha tênue entre otimização dos processos administrativos e possíveis transgressões em relação aos direitos fundamentais. Nesse sentido, técnicas de hacking governamental, ou seja, de superação de recursos de segurança em dispositivos pessoais, vem ganhando uma escalabilidade crescente e envolve a ampliação de fabricantes, revendedores e contratos com a administração pública, ao passo em que seus efeitos colaterais aos direitos fundamentais, sobretudo em relação à sociedade civil, vêm sendo denunciados internacionalmente.", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "place": "Recife, PE", + "publisher": "IP.rec", + "shortTitle": "Mercadores da Insegurança", + "attachments": [], + "tags": [ + { + "tag": "Digital" + }, + { + "tag": "Direito" + }, + { + "tag": "Governamental" + }, + { + "tag": "Insegurança" + }, + { + "tag": "dados" + }, + { + "tag": "hacking" + }, + { + "tag": "vazamento" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "8532511015" + }, + "items": [ + { + "itemType": "book", + "title": "Harry Potter E a Pedra Filosofal", + "creators": [ + { + "firstName": "J. K.", + "lastName": "Rowling", + "creatorType": "author" + } + ], + "date": "2000-05-29", + "ISBN": "9788532511010", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "publisher": "Rocco", + "attachments": [], + "tags": [ + { + "tag": "Biblioteconomia e ciência da informação" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9786555320275" + }, + "items": [ + { + "itemType": "book", + "title": "Harry Potter e o Prisioneiro de Azkaban", + "creators": [ + { + "firstName": "J. K.", + "lastName": "Rowling", + "creatorType": "author" + }, + { + "firstName": "Lia", + "lastName": "Wyler", + "creatorType": "contributor" + }, + { + "firstName": "Arch", + "lastName": "Apolar", + "creatorType": "contributor" + } + ], + "date": "2020-03-04", + "ISBN": "9786555320275", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "publisher": "Rocco", + "series": "Harry Potter", + "attachments": [], + "tags": [ + { + "tag": "Harry-Potter" + }, + { + "tag": "Literatura infanto-juvenil" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9786587233956" + }, + "items": [ + { + "itemType": "book", + "title": "Einstein Socialista: Entrevistas, manifestos e artigos do maior cientista do século XX", + "creators": [ + { + "firstName": "Albert", + "lastName": "Einstein", + "creatorType": "author" + }, + { + "firstName": "Hugo", + "lastName": "Albuquerque", + "creatorType": "editor" + }, + { + "firstName": "Lígia Magalhães", + "lastName": "Marinho", + "creatorType": "translator" + } + ], + "date": "2023-04-12", + "ISBN": "9786587233956", + "abstractNote": "Por serem supostamente purgadas da “ideologia”, as ciências exatas alcançaram entre nós um status de demasiada confiabilidade, objetividade e verdade, Entretanto, Albert Einstein, o maior físico do século XX, não compactuava com essa crença e defendia abertamente os valores socialistas, colocando a própria narrativa cientificista, quase sempre favorável à ordem capitalista, em curto-circuito.\n\nEinstein era um militante e não se calou diante das falaciosas equiparações entre a Alemanha Nazista e a União Soviética. Nem se calou, como judeu, diante das violências cometidas contra os palestinos, pouco depois do Holocausto, pelos colonos judeus no nascente Estado de Israel. Tampouco poupou críticas à segregação racial nos Estados Unidos, onde foi lecionar em seus últimos anos.\n\nQuando a Guerra Fria estava a todo vapor, Einstein escreveu “Por que o Socialismo?” –, um de seus artigos mais conhecidos sobre política e frequentemente esquecido e dissociado de sua imagem. Não à toa, este texto, vez ou outra, é apresentado como “novidade” e não cansa de surpreender geração após geração. E que não se diga que era uma forma atenuada de socialismo que Einstein estava falando:\n\n“Numa economia planificada, em que a produção é ajustada às necessidades da comunidade, o trabalho a ser feito seria distribuído entre todas as pessoas aptas ao trabalho e garantiria condições de vida a todo homem, mulher e criança.”\n\nOu mesmo que, especificamente sobre a Revolução Russa, ele tenha confessado a Viereck que: \n\n“O bolchevismo é uma experiência extraordinária. Não é impossível que a deriva da evolução social daqui para a frente seja em direção ao comunismo. O experimento bolchevista talvez valha a pena.”\n\nCom uma certa dose de utopismo e um enorme enigma de como o socialismo pode ser alcançado e mantido, Einstein Socialista nos apresenta uma série de artigos, entrevistas e manifestos que revelam um lado muitas vezes negligenciado e “esquecido” de um dos maiores cientistas do mundo.", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "numPages": "96", + "place": "São Paulo, SP", + "publisher": "Autonomia Literaria", + "shortTitle": "Einstein Socialista", + "attachments": [], + "tags": [ + { + "tag": "Ciências sociais" + }, + { + "tag": "einstein" + }, + { + "tag": "nazismo" + }, + { + "tag": "relatividade" + }, + { + "tag": "socialismo" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9788565053082" + }, + "items": [ + { + "itemType": "book", + "title": "State of the Art in Health and Knowledge", + "creators": [ + { + "firstName": "Sociedade Beneficente Israelita Brasileira Albert", + "lastName": "Einstein", + "creatorType": "author" + }, + { + "firstName": "Juliana", + "lastName": "Samel", + "creatorType": "translator" + } + ], + "date": "2023-02-02", + "ISBN": "9788565053082", + "abstractNote": "The book presents the Albert Einstein Teaching and Research Center - Campus Cecilia and Abram Szajman, an architectural work in the city of São Paulo, created to be one of the most advanced teaching and research centers in the world. Students and researchers interact in this building to generate knowledge, with the aim of boosting Brazilian research and all this in an environment integrated with greenery, with the use of natural light and renewable energy technology.", + "language": "Inglês (EUA)", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "numPages": "58", + "place": "São Paulo, SP", + "publisher": "Sociedade Beneficente Israelita Brasileira Albert Einstein", + "attachments": [], + "tags": [ + { + "tag": "Arquitetura" + }, + { + "tag": "Landscaping" + }, + { + "tag": "architecture" + }, + { + "tag": "health" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9786580341221" + }, + "items": [ + { + "itemType": "book", + "title": "Filipson: Memórias de uma menina na primeira colônia judaica no Rio Grande do Sul (1904-1920)", + "creators": [ + { + "firstName": "Frida", + "lastName": "Alexandr", + "creatorType": "author" + }, + { + "firstName": "Regina", + "lastName": "Zilberman", + "creatorType": "contributor" + } + ], + "date": "2023-05-11", + "ISBN": "9786580341221", + "abstractNote": "“Já ouviram falar de Filipson? Um nome esquisito. Nem parece brasileiro. Mas, dentro do Brasil imenso, constituía um pontinho minúsculo que ficava lá nas bandas do Sul, perdido no meio de diversas colônias prósperas compostas em sua maioria de imigrantes espanhóis, italianos e alemães e uma ou outra fazenda de brasileiros.”\n\nDesde a primeira linha, Frida Alexandr surpreende o leitor, interpelando-o com uma pergunta. Mesmo em 1967, quando suas memórias foram publicadas em edição restrita, provavelmente poucos responderiam afirmativamente à sua questão.\n\nFilipson foi a primeira colônia judaica oficial do Brasil, formada por imigrantes judeus provenientes da Bessarábia (na região onde atualmente se localiza a Moldávia). Os pais e irmãos mais velhos de Frida chegaram ao Brasil com o grupo pioneiro, em 1904, e em \"Filipson: memórias de uma menina na primeira colônia judaica no Rio Grande do Sul (1904-1920)\". Frida deixa um registro que vai dos primeiros dias da colônia à melancólica despedida, em 1920, quando sua família decide partir novamente.\n\nEntre os dois pontos, desliza a memória de Frida, que organiza os fatos sem a preocupação de ordená-los no tempo. O importante é como essas cenas — que envolvem seus familiares, sua passagem pela escola, as dificuldades financeiras da família, as ameaças representadas por uma natureza nem sempre hospitaleira — repercutem em sua sensibilidade. Frida se vale da linguagem para transmitir a emoção na forma como a vivenciou.\n\n\"Filipson\", com posfácio da pesquisadora e escritora Regina Zilberman, é um testemunho de uma etapa do processo de adaptação e preservação dos judeus do leste da Europa no Brasil. Mas esse caráter documental é acompanhado pela recuperação sensível daqueles momentos fundadores, como se a autora, à maneira de Proust, fosse em busca das vivências daquele tempo, para transmiti-lo a um leitor que pouco conhece sobre o período.", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "numPages": "360", + "place": "SÃO PAULO, SP", + "publisher": "Chão Editora", + "shortTitle": "Filipson", + "attachments": [], + "tags": [ + { + "tag": "Biografias" + }, + { + "tag": "judeus" + }, + { + "tag": "memórias" + }, + { + "tag": "mulheres" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9786555250053" + }, + "items": [ + { + "itemType": "book", + "title": "Ilíada = Ἰλιάς", + "creators": [ + { + "firstName": "", + "lastName": "Homero", + "creatorType": "author", + "fieldMode": 1 + }, + { + "firstName": "Trajano", + "lastName": "Vieira", + "creatorType": "translator" + } + ], + "date": "2020-03-23", + "ISBN": "9786555250053", + "abstractNote": "Composta no século VIII a.C., a Ilíada é considerada o marco inaugural da literatura ocidental. Tradicionalmente atribuída a Homero, a obra aborda o período de algumas semanas no último ano da Guerra de Troia, durante o cerco final dos contingentes gregos à cidadela do rei Príamo, na Ásia Menor. Com seus mais de 15 mil versos, a Ilíada ganha agora uma nova tradução — das mãos de Trajano Vieira, professor livre-docente da Unicamp e premiado tradutor da Odisseia —, rigorosamente metrificada, que busca recriar em nossa língua a excelência do original, com seus símiles e invenções vocabulares. A presente edição, bilíngue, traz ainda uma série de aparatos, como um índice onomástico completo, um posfácio do tradutor, excertos da crítica, e o célebre ensaio de Simone Weil, “A Ilíada ou o poema da força”.", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "numPages": "1048", + "place": "São Paulo, SP", + "publisher": "Editora 34", + "attachments": [], + "tags": [ + { + "tag": "Literatura grega" + }, + { + "tag": "Literatura." + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9786556752631" + }, + "items": [ + { + "itemType": "book", + "title": "Manual de Sentença Trabalhista: Compreendendo a técnica da setença trabalhista para concurso", + "creators": [ + { + "firstName": "Aline", + "lastName": "Leporaci", + "creatorType": "author" + }, + { + "firstName": "Adriana Leandro de Sousa", + "lastName": "Freitas", + "creatorType": "contributor" + } + ], + "date": "2023-03-02", + "ISBN": "9786556752631", + "abstractNote": "Nesse livro sobre sentença trabalhista, fase tão concorrida do concurso para a Magistratura do Trabalho, procuramos trazer os aspectos mais importantes a serem observados pelo candidato. O leitor poderá verificar a ordem de julgamento a seguir e a importância da fixação da prejudicialidade entre as matérias a serem analisadas. Além disso, também aprenderá as técnicas de distribuição do ônus da prova, e suas diversas teorias, sempre ressaltando qual deva ser de aplicação preferencial pelo candidato. O livro traz diversos aspectos teóricos, que são essenciais para a preparação de todos os interessados em efetivamente aprender a técnica da elaboração da sentença trabalhista, sempre com leitura fácil e direta. E não nos esquecemos dos aspectos práticos, pois o leitor terá exercícios de fixação de jornada de trabalho, e sentenças inéditas elaboradas pelas Autoras, com os respectivos gabaritos e sugestão de redação.", + "edition": "2", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "numPages": "232", + "place": "Rio de Janeiro, RJ", + "publisher": "Freitas Bastos Editora", + "shortTitle": "Manual de Sentença Trabalhista", + "attachments": [], + "tags": [ + { + "tag": "Direito" + }, + { + "tag": "Elaboração" + }, + { + "tag": "Jornada" + }, + { + "tag": "Magistratura" + }, + { + "tag": "Trabalhista" + }, + { + "tag": "Trabalho" + }, + { + "tag": "Técnica" + }, + { + "tag": "concurso" + }, + { + "tag": "juiz" + }, + { + "tag": "modelos" + }, + { + "tag": "sentença" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9786559602513" + }, + "items": [ + { + "itemType": "book", + "title": "Batman", + "creators": [ + { + "firstName": "John", + "lastName": "Ridley", + "creatorType": "author" + }, + { + "firstName": "James Tynion", + "lastName": "IV", + "creatorType": "contributor" + }, + { + "firstName": "Dandara", + "lastName": "Palankof", + "creatorType": "contributor" + }, + { + "firstName": "Pedro", + "lastName": "Catarino", + "creatorType": "contributor" + }, + { + "firstName": "Travel", + "lastName": "Foreman", + "creatorType": "contributor" + }, + { + "firstName": "Riccardo", + "lastName": "Federici", + "creatorType": "contributor" + }, + { + "firstName": "Jorge", + "lastName": "Jimenez", + "creatorType": "contributor" + } + ], + "date": "2022-03-11", + "ISBN": "9786559602513", + "abstractNote": "Aventuras do Batman", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "numPages": "100", + "place": "Barueri, SP", + "publisher": "Panini Comics", + "attachments": [], + "tags": [ + { + "tag": "Cartoons; caricaturas e quadrinhos" + }, + { + "tag": "quadrinhos" + }, + { + "tag": "super-herois" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9786559605101" + }, + "items": [ + { + "itemType": "book", + "title": "Superman", + "creators": [ + { + "firstName": "Sean", + "lastName": "Lewis", + "creatorType": "author" + }, + { + "firstName": "Phillip Kennedy", + "lastName": "Johnson", + "creatorType": "contributor" + }, + { + "firstName": "Gabriel", + "lastName": "Faria", + "creatorType": "contributor" + }, + { + "firstName": "Rodrigo", + "lastName": "Barros", + "creatorType": "contributor" + }, + { + "firstName": "Sami", + "lastName": "Basri", + "creatorType": "contributor" + }, + { + "firstName": "Phil", + "lastName": "Hester", + "creatorType": "contributor" + }, + { + "firstName": "Daniel", + "lastName": "Sampere", + "creatorType": "contributor" + } + ], + "date": "2022-03-10", + "ISBN": "9786559605101", + "abstractNote": "Aventuras do Superman", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "numPages": "100", + "place": "Barueri, SP", + "publisher": "Panini Comics", + "attachments": [], + "tags": [ + { + "tag": "Cartoons; caricaturas e quadrinhos" + }, + { + "tag": "quadrinhos" + }, + { + "tag": "super-herois" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "8576664984" + }, + "items": [ + { + "itemType": "book", + "title": "A Religião Nos Limites Da Simples Razão", + "creators": [], + "date": "2006-01-02", + "ISBN": "9788576664987", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "publisher": "Escala Educacional", + "series": "Série Filosofar", + "attachments": [], + "tags": [ + { + "tag": "Literatura" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "search", + "input": { + "ISBN": "9788591597512" + }, + "items": [ + { + "itemType": "book", + "title": "Visões da áfrica: angola e moçambiqueJorge Alves de Lima Filho", + "creators": [ + { + "firstName": "Jorge Alves de", + "lastName": "Lima Filho", + "creatorType": "author" + } + ], + "date": "2015-09-29", + "ISBN": "9788591597512", + "language": "pt-BR", + "libraryCatalog": "Câmara Brasileira do Livro ISBN", + "publisher": "Jorge Alves de Lima Filho", + "shortTitle": "Visões da áfrica", + "attachments": [], + "tags": [ + { + "tag": "Coleções de obras diversas sem assunto específico" + } + ], + "notes": [], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/Cambridge Engage Preprints.js b/Cambridge Engage Preprints.js index 82c29cb314a..f2127e9338b 100644 --- a/Cambridge Engage Preprints.js +++ b/Cambridge Engage Preprints.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-12-08 18:55:55" + "lastUpdated": "2023-10-23 08:35:07" } /* @@ -52,7 +52,7 @@ function detectWeb(doc, url) { function getSearchResults(doc, checkOnly) { var items = {}; var found = false; - var rows = doc.querySelectorAll('.MatchResult > a'); + var rows = doc.querySelectorAll('.MatchResult article a'); for (let row of rows) { let href = row.href; @@ -68,11 +68,9 @@ function getSearchResults(doc, checkOnly) { async function doWeb(doc, url) { if (detectWeb(doc, url) == 'multiple') { let items = await Zotero.selectItems(getSearchResults(doc, false)); - if (items) { - await Promise.all( - Object.keys(items) - .map(url => requestDocument(url).then(scrape)) - ); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } } else { @@ -89,6 +87,9 @@ async function scrape(doc, url = doc.location.href) { translator.setHandler('itemDone', (_obj, item) => { item.publisher = attr(doc, 'meta[property="og:site_name"]', 'content'); item.libraryCatalog = "Cambridge Engage Preprints"; + if (item.date) { + item.date = ZU.strToISO(item.date); + } item.complete(); }); @@ -102,6 +103,7 @@ var testCases = [ { "type": "web", "url": "https://chemrxiv.org/engage/chemrxiv/search-dashboard?text=acid", + "defer": true, "items": "multiple" }, { @@ -123,7 +125,7 @@ var testCases = [ "creatorType": "author" } ], - "date": "2021/10/08", + "date": "2021-10-08", "DOI": "10.26434/chemrxiv-2021-mjpkz", "abstractNote": "Conversion of readily available feedstocks to valuable platform chemicals via a sustainable catalytic pathway has always been one of the key focuses of synthetic chemists. Cheaper, less toxic, and more abundant base metals as a catalyst for performing such transformations provide an additional boost. In this context, herein, we report a reformation of readily available feedstock, ethylene glycol, to value-added platform molecules, glycolic acid, and lactic acid. A bench stable base metal complex {[HN(C2H4PPh2)2]Mn(CO)2Br}, Mn-I, known as Mn-PhMACHO, catalyzed the reformation of ethylene glycol to glycolic acid at 140 oC in high selectivity with a turnover number TON = 2400, surpassing previously used homogeneous catalysts for such a reaction. Pure hydrogen gas is evolved without the need for an acceptor. On the other hand, a bench stable Mn(I)-complex, {(iPrPN5P)Mn(CO)2Br}, Mn-III, with a triazine backbone, efficiently catalyzed the acceptorless dehydrogenative coupling of ethylene glycol and methanol for the synthesis of lactic acid, even at a ppm level of catalyst loading, reaching the TON of 11,500. Detailed mechanistic studies were performed to elucidate the involvements of different manganese(I)-species during the catalysis.", "language": "en", @@ -180,7 +182,7 @@ var testCases = [ "creatorType": "author" } ], - "date": "2019/10/02", + "date": "2019-10-03", "DOI": "10.33774/apsa-2019-if2he-v2", "abstractNote": "The discipline of political science has been engaged in discussion about when, why, and how to make scholarship more transparent for at least three decades. This piece argues that qualitative researchers can achieve transparency in diverse ways, using techniques and strategies that allow them to balance and optimize among competing considerations that affect the pursuit of transparency.. We begin by considering the “state of the debate,” briefly outlining the contours of the scholarship on transparency in political and other social sciences, which so far has focussed mostly on questions of “whether” and “what” to share. We investigate competing considerations that researchers have to consider when working towards transparent research. The heart of the piece considers various strategies, illustrated by exemplary applications, for making qualitative research more transparent.", "language": "en", @@ -238,7 +240,7 @@ var testCases = [ "creatorType": "author" } ], - "date": "2021/07/20", + "date": "2021-07-20", "DOI": "10.33774/miir-2021-7cdx1", "abstractNote": "This report addresses the construction of carbon fibre wing boxes and the problems associated with using carbon fibre sheets rather than individual carbon fibre tapes. In the case that the wing boxes are developable surfaces the lay up of carbon fibre sheets is straightforward, since the fibres can follow the contours of the surface without any need for shearing or extension of the fibres. To further expand the potential design space for the wing boxes, this report investigates the lay up of sheets over non-developable surfaces where some shearing of the sheet is required to achieve the desired results. In this report, three analytical approaches are considered, driven by the results from numerical studies on different surface geometries. Each of the approaches offers insights as to the type of geometric perturbations achievable when constrained by a maximum shear angle.", "language": "en", diff --git a/Climate Change and Human Health Literature Portal.js b/Climate Change and Human Health Literature Portal.js index 3ee89a2bcd6..ed4b318fe43 100644 --- a/Climate Change and Human Health Literature Portal.js +++ b/Climate Change and Human Health Literature Portal.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-11-10 02:47:43" + "lastUpdated": "2023-08-22 04:14:33" } /* @@ -40,9 +40,16 @@ function detectWeb(doc, url) { if (url.includes('/index.cfm/detail/')) { return 'journalArticle'; } - else if (getSearchResults(doc, true)) { + + let appRoot = doc.querySelector("#app"); // Ajax app "mount point" + if (appRoot) { + // Watch for live filtering of search results) + Z.monitorDOMChanges(appRoot); + } + if (getSearchResults(doc, true)) { return 'multiple'; } + return false; } @@ -64,11 +71,9 @@ function getSearchResults(doc, checkOnly) { async function doWeb(doc, url) { if (detectWeb(doc, url) == 'multiple') { let items = await Zotero.selectItems(getSearchResults(doc, false)); - if (items) { - await Promise.all( - Object.keys(items) - .map(url => requestDocument(url).then(scrape)) - ); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } } else { @@ -76,7 +81,7 @@ async function doWeb(doc, url) { } } -function scrape(doc) { +async function scrape(doc) { var pmid = text(doc, 'li>a[href*="www.ncbi.nlm.nih.gov/pubmed"]'); var doi = text(doc, 'li>a[href*="doi.org/10."]'); var abstract = text(doc, '#cchh-detail-abstract'); @@ -185,6 +190,7 @@ var testCases = [ { "type": "web", "url": "https://tools.niehs.nih.gov/cchhl/index.cfm/main/search#/params?searchTerm=heat%20pump&selectedFacets=&selectedResults=", + "defer": true, "items": "multiple" }, { diff --git a/DOI Content Negotiation.js b/DOI Content Negotiation.js index 2a83e024012..fa4e9a090b8 100644 --- a/DOI Content Negotiation.js +++ b/DOI Content Negotiation.js @@ -3,13 +3,12 @@ "label": "DOI Content Negotiation", "creator": "Sebastian Karcher", "target": "", - "minVersion": "4.0.29.11", + "minVersion": "5.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 8, - "browserSupport": "gcs", - "lastUpdated": "2020-04-20 20:04:00" + "lastUpdated": "2023-09-22 09:54:11" } /* @@ -58,145 +57,255 @@ function filterQuery(items) { return dois; } -function doSearch(items) { - var dois = filterQuery(items); - if (!dois.length) return; - processDOIs(dois); +async function doSearch(items) { + for (let doi of filterQuery(items)) { + await processDOI(doi); + } } -function processDOIs(dois) { - var doi = dois.pop(); +async function processDOI(doi) { + let response = await requestText( + `https://doi.org/${encodeURIComponent(doi)}`, + { headers: { Accept: "application/vnd.datacite.datacite+json, application/vnd.crossref.unixref+xml, application/vnd.citationstyles.csl+json" } } + ); // by content negotiation we asked for datacite or crossref format, or CSL JSON - ZU.doGet('https://doi.org/' + encodeURIComponent(doi), function (text) { - if (!text) { - return; - } - Z.debug(text); - - var trans = Zotero.loadTranslator('import'); - trans.setString(text); - if (text.includes("Other\nLe code est accompagné de commentaires de F. A. Vogel, qui signe l'épitre dédicatoire

    Other

    \nReliure 18è siècle

    Other

    \nEx-libris manuscrit \"Ex libris Dufour\"" + ] + } + ] + }, + { + "type": "search", + "input": { + "DOI": "10.7336/academicus.2014.09.05" }, - { - "firstName": "", - "lastName": "Université De Lorraine-Direction De La Documentation Et De L'Edition", - "creatorType": "contributor" - } - ], - "tags": [ - "Droit" - ], - "relations": [], - "attachments": [], - "notes": [ - "

    Other

    \nLe code est accompagné de commentaires de F. A. Vogel, qui signe l'épitre dédicatoire

    Other

    \nReliure 18è siècle

    Other

    \nEx-libris manuscrit \"Ex libris Dufour\"" + "items": [ + { + "itemType": "journalArticle", + "title": "Second world war, communism and post-communism in Albania, an equilateral triangle of a tragic trans-Adriatic story. The Eftimiadi’s Saga", + "creators": [ + { + "firstName": "Muner", + "lastName": "Paolo", + "creatorType": "author" + } + ], + "date": "01/2014", + "DOI": "10.7336/academicus.2014.09.05", + "ISSN": "20793715", + "accessDate": "2019-02-02T03:28:48Z", + "libraryCatalog": "DOI.org (Crossref)", + "pages": "69-78", + "publicationTitle": "Academicus International Scientific Journal", + "relations": [], + "url": "http://academicus.edu.al/?subpage=volumes&nr=9", + "volume": "9", + "attachments": [], + "tags": [], + "notes": [] + } ] - }] -}, -{ - "type": "search", - "input": { - "DOI": "10.7336/academicus.2014.09.05" }, - "items": [{ - "itemType": "journalArticle", - "url": "http://academicus.edu.al/?subpage=volumes&nr=9", - "volume": "9", - "pages": "69-78", - "publicationTitle": "Academicus International Scientific Journal", - "ISSN": "20793715", - "date": "01/2014", - "DOI": "10.7336/academicus.2014.09.05", - "accessDate": "2019-02-02T03:28:48Z", - "libraryCatalog": "DOI.org (Crossref)", - "title": "Second world war, communism and post-communism in Albania, an equilateral triangle of a tragic trans-Adriatic story. The Eftimiadi’s Saga", - "creators": [{ - "firstName": "Muner", - "lastName": "Paolo", - "creatorType": "author" - }], - "tags": [], - "relations": [], - "attachments": [], - "notes": [] - }] -} + { + "type": "search", + "input": [ + { + "DOI": "10.5555/12345678" + }, + { + "DOI": "10.1109/TPS.1987.4316723" + }, + { + "DOI": "10.5555/666655554444" + } + ], + "items": [ + { + "itemType": "journalArticle", + "title": "Toward a Unified Theory of High-Energy Metaphysics: Silly String Theory", + "creators": [ + { + "creatorType": "author", + "firstName": "Josiah", + "lastName": "Carberry" + }, + { + "creatorType": "contributor", + "fieldMode": 1, + "lastName": "Friends of Josiah Carberry" + } + ], + "date": "2008-08-14", + "DOI": "10.5555/12345678", + "ISSN": "0264-3561", + "abstractNote": "The characteristic theme of the works of Stone is the bridge between culture and society. Several narratives concerning the fatal !aw, and subsequent dialectic, of semioticist class may be found. Thus, Debord uses the term ‘the subtextual paradigm of consensus’ to denote a cultural paradox. The subject is interpolated into a neocultural discourse that includes sexuality as a totality. But Marx’s critique of prepatriarchialist nihilism states that consciousness is capable of signi\"cance. The main theme of Dietrich’s[1]model of cultural discourse is not construction, but neoconstruction. Thus, any number of narratives concerning the textual paradigm of narrative exist. Pretextual cultural theory suggests that context must come from the collective unconscious.", + "issue": "11", + "journalAbbreviation": "Journal of Psychoceramics", + "language": "en", + "libraryCatalog": "DOI.org (Crossref)", + "pages": "1-3", + "publicationTitle": "Journal of Psychoceramics", + "shortTitle": "Toward a Unified Theory of High-Energy Metaphysics", + "url": "https://ojs33.crossref.publicknowledgeproject.org/index.php/test/article/view/2", + "volume": "5", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + }, + { + "itemType": "journalArticle", + "title": "Bulk and Surface Plasmons in Artificially Structured Materials", + "creators": [ + { + "creatorType": "author", + "firstName": "John J.", + "lastName": "Quinn" + }, + { + "creatorType": "author", + "firstName": "Josiah S.", + "lastName": "Carberry" + } + ], + "date": "1987", + "DOI": "10.1109/TPS.1987.4316723", + "ISSN": "0093-3813", + "issue": "4", + "journalAbbreviation": "IEEE Trans. Plasma Sci.", + "libraryCatalog": "DOI.org (Crossref)", + "pages": "394-410", + "publicationTitle": "IEEE Transactions on Plasma Science", + "url": "http://ieeexplore.ieee.org/document/4316723/", + "volume": "15", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + }, + { + "itemType": "journalArticle", + "title": "The Memory Bus Considered Harmful", + "creators": [ + { + "creatorType": "author", + "firstName": "Josiah", + "lastName": "Carberry" + } + ], + "date": "2012-10-11", + "DOI": "10.5555/666655554444", + "ISSN": "0264-3561", + "issue": "11", + "journalAbbreviation": "Journal of Psychoceramics", + "language": "en", + "libraryCatalog": "DOI.org (Crossref)", + "pages": "1-3", + "publicationTitle": "Journal of Psychoceramics", + "url": "https://ojs33.crossref.publicknowledgeproject.org/index.php/test/article/view/8", + "volume": "9", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + } ] /** END TEST CASES **/ diff --git a/DOI.js b/DOI.js index dba25c54a0d..1637c397de5 100644 --- a/DOI.js +++ b/DOI.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-03-12 01:46:07" + "lastUpdated": "2023-10-18 11:11:59" } /* @@ -151,7 +151,7 @@ function detectWeb(doc, url) { return "journalArticle"; // A decent guess } -function retrieveDOIs(doiOrDOIs) { +async function retrieveDOIs(doiOrDOIs) { let showSelect = Array.isArray(doiOrDOIs); let dois = showSelect ? doiOrDOIs : [doiOrDOIs]; let items = {}; @@ -214,14 +214,14 @@ function retrieveDOIs(doiOrDOIs) { // Don't throw on error translate.setHandler("error", function () {}); - translate.translate(); + await translate.translate(); } } -function doWeb(doc, url) { +async function doWeb(doc, url) { let doiOrDOIs = getDOIs(doc, url); Z.debug(doiOrDOIs); - retrieveDOIs(doiOrDOIs); + await retrieveDOIs(doiOrDOIs); } /** BEGIN TEST CASES **/ diff --git a/DSpace Intermediate Metadata.js b/DSpace Intermediate Metadata.js new file mode 100644 index 00000000000..3f02ce18f3c --- /dev/null +++ b/DSpace Intermediate Metadata.js @@ -0,0 +1,208 @@ +{ + "translatorID": "2c05e2d1-a533-448f-aa20-e919584864cb", + "label": "DSpace Intermediate Metadata", + "creator": "Sebastian Karcher", + "target": "xml", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 1, + "lastUpdated": "2022-12-24 19:29:02" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2022 Sebastian Karcher + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +function detectImport() { + var text = Zotero.read(1000); + return text.includes("http://www.dspace.org/xmlns/dspace/dim"); +} + +function getType(string) { + string = string.toLowerCase(); + if (string.includes("book_section") || string.includes("chapter")) { + return "bookSection"; + } + else if (string.includes("book") || string.includes("monograph")) { + return "book"; + } + else if (string.includes("report")) { + return "report"; + } + else if (string.includes("proceedings") || string.includes("conference")) { + return "conferencePaper"; + } + else { + return "journalArticle"; // default -- most of the catalog + } +} +function doImport() { + var xml = Zotero.getXML(); + var ns = { + dim: 'http://www.dspace.org/xmlns/dspace/dim', + mets: 'http://www.loc.gov/METS/', + xlink: 'http://www.w3.org/TR/xlink/' + }; + + let type = ZU.xpathText(xml, '//dim:field[@element="type"]', ns); + var item = new Zotero.Item("journalArticle"); + if (type) { + item.itemType = getType(type); + } + + item.title = ZU.trimInternal(ZU.xpathText(xml, '//dim:field[@element="title"]', ns)); + let abstract = ZU.xpath(xml, '//dim:field[@qualifier="abstract"]', ns); + if (abstract.length) { + item.abstractNote = abstract[0].textContent; + } + item.date = ZU.xpathText(xml, '//dim:field[@element="date" and @qualifier="issued"]', ns); + item.language = ZU.xpathText(xml, '//dim:field[@element="language"]', ns); + item.issue = ZU.xpathText(xml, '//dim:field[@element="bibliographicCitation" and @qualifier="issue"]', ns); + item.volume = ZU.xpathText(xml, '//dim:field[@element="bibliographicCitation" and @qualifier="volume"]', ns); + item.publicationTitle = ZU.xpathText(xml, '//dim:field[@element="title" and @qualifier="parent"]', ns); + if (!item.publicationTitle) { + item.publicationTitle = ZU.xpathText(xml, '//dim:field[@element="bibliographicCitation" and @qualifier="title"]', ns); + } + item.conferenceName = ZU.xpathText(xml, '//dim:field[@element="bibliographicCitation" and @qualifier="conferencename"]', ns); + item.publisher = ZU.xpathText(xml, '//dim:field[@element="publisher" and not(@qualifier="place")]', ns); + item.place = ZU.xpathText(xml, '//dim:field[@element="publisher" and @qualifier="place"]', ns); + item.series = ZU.xpathText(xml, '//dim:field[@element="relation" and @qualifier="ispartofseries"]', ns); + item.ISSN = ZU.xpathText(xml, '//dim:field[@element="identifier" and @qualifier="issn"]', ns); + let pages = ZU.xpathText(xml, '//dim:field[@element="format" and @qualifier="pagerange"]', ns); + if (pages) { + item.pages = pages.replace(/pp?\./i, ""); + } + else if (ZU.xpathText(xml, '//dim:field[@element="bibliographicCitation" and @qualifier="stpage"]', ns)) { + item.pages = ZU.xpathText(xml, '//dim:field[@element="bibliographicCitation" and @qualifier="stpage"]', ns) + + "-" + ZU.xpathText(xml, '//dim:field[@element="bibliographicCitation" and @qualifier="endpage"]', ns); + } + let numPages = ZU.xpathText(xml, '//dim:field[@element="format" and @qualifier="pages"]', ns); + if (numPages) { + item.numPages = numPages.replace(/pp?\.?/i, ""); + } + item.url = ZU.xpathText(xml, '//dim:field[@element="identifier" and @qualifier="uri"]', ns); // using the handle + + let authors = ZU.xpath(xml, '//dim:field[@element="contributor" and @qualifier="author"]', ns); + for (let author of authors) { + item.creators.push(ZU.cleanAuthor(author.textContent, "author", true)); + } + + let tags = ZU.xpath(xml, '//dim:field[@element="subject"]', ns); + for (let tag of tags) { + item.tags.push(tag.textContent); + } + // getting only the first PDF + let pdfURL = ZU.xpathText(xml, '//mets:file[@MIMETYPE="application/pdf"][1]/mets:FLocat/@xlink:href', ns); + if (pdfURL) { + item.attachments.push({ url: pdfURL, title: "Full Text PDF", mimeType: "application/pdf" }); + } + item.complete(); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "import", + "input": "\r\n \r\n \r\n \r\n \r\n \r\n Schittone, Joe\r\n Franklin, Erik C.\r\n Hudson, J. Harold\r\n Anderson, Jeff\r\n 2021-06-24T15:18:40Z\r\n 2021-06-24T15:18:40Z\r\n 2006\r\n http://hdl.handle.net/1834/20117\r\n This document presents the results of the monitoring of a repaired coral reef injured by the M/V Connected vessel grounding incident of March 27, 2001. This groundingoccurred in Florida state waters within the boundaries of the Florida Keys National Marine Sanctuary (FKNMS). The National Oceanic and Atmospheric Administration (NOAA) and the Board of Trustees of the Internal Improvement Trust Fund of the State of Florida, (“State of Florida” or “state”) are the co-trustees for the natural resourceswithin the FKNMS and, thus, are responsible for mediating the restoration of the damaged marine resources and monitoring the outcome of the restoration actions. Therestoration monitoring program tracks patterns of biological recovery, determines the success of restoration measures, and assesses the resiliency to environmental andanthropogenic disturbances of the site over time.The monitoring program at the Connected site was to have included an assessment of the structural stability of installed restoration modules and biological condition of reattached corals performed on the following schedule: immediately (i.e., baseline), 1, 3, and 6 years after restoration and following a catastrophic event. Restoration of this site was completed on July 20, 2001. Due to unavoidable delays in the settlement of the case, the“baseline” monitoring event for this site occurred in July 2004. The catastrophic monitoring event occurred on August 31, 2004, some 2 ½ weeks after the passage of Hurricane Charley which passed nearby, almost directly over the Dry Tortugas. In September 2005, the year one monitoring event occurred shortly after the passage of Hurricane Katrina, some 70 km to the NW. This report presents the results of all three monitoring events. (PDF contains 37 pages.)\r\n application/pdf\r\n application/pdf\r\n en\r\n NOAA/National Ocean Service/National Marine Sanctuary Program\r\n Marine Sanctuaries Conservation Series\r\n http://sanctuaries.noaa.gov/science/conservation/pdfs/connected.pdf\r\n Ecology\r\n Management\r\n Environment\r\n Florida Keys National Marine Sanctuary\r\n Coral\r\n Grounding\r\n Restoration\r\n Monitoring\r\n Hurricane Charley\r\n Hurricane Katrina\r\n Acropora palmata\r\n M/V CONNECTED Coral Reef Restoration Monitoring Report,\r\n Monitoring Events 2004-2005. Florida Keys National Marine Sanctuary Monroe County, Florida\r\n monograph\r\n NMSP-0\r\n Silver Spring, MD\r\n 2021-06-24T15:18:40Z\r\n http://aquaticcommons.org/id/eprint/2312\r\n 403\r\n 2011-09-29 19:16:51\r\n 2312\r\n United States National Ocean Service\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n ", + "items": [ + { + "itemType": "book", + "title": "M/V CONNECTED Coral Reef Restoration Monitoring Report, Monitoring Events 2004-2005. Florida Keys National Marine Sanctuary Monroe County, Florida", + "creators": [ + { + "firstName": "Joe", + "lastName": "Schittone", + "creatorType": "author" + }, + { + "firstName": "Erik C.", + "lastName": "Franklin", + "creatorType": "author" + }, + { + "firstName": "J. Harold", + "lastName": "Hudson", + "creatorType": "author" + }, + { + "firstName": "Jeff", + "lastName": "Anderson", + "creatorType": "author" + } + ], + "date": "2006", + "abstractNote": "This document presents the results of the monitoring of a repaired coral reef injured by the M/V Connected vessel grounding incident of March 27, 2001. This groundingoccurred in Florida state waters within the boundaries of the Florida Keys National Marine Sanctuary (FKNMS). The National Oceanic and Atmospheric Administration (NOAA) and the Board of Trustees of the Internal Improvement Trust Fund of the State of Florida, (“State of Florida” or “state”) are the co-trustees for the natural resourceswithin the FKNMS and, thus, are responsible for mediating the restoration of the damaged marine resources and monitoring the outcome of the restoration actions. Therestoration monitoring program tracks patterns of biological recovery, determines the success of restoration measures, and assesses the resiliency to environmental andanthropogenic disturbances of the site over time.The monitoring program at the Connected site was to have included an assessment of the structural stability of installed restoration modules and biological condition of reattached corals performed on the following schedule: immediately (i.e., baseline), 1, 3, and 6 years after restoration and following a catastrophic event. Restoration of this site was completed on July 20, 2001. Due to unavoidable delays in the settlement of the case, the“baseline” monitoring event for this site occurred in July 2004. The catastrophic monitoring event occurred on August 31, 2004, some 2 ½ weeks after the passage of Hurricane Charley which passed nearby, almost directly over the Dry Tortugas. In September 2005, the year one monitoring event occurred shortly after the passage of Hurricane Katrina, some 70 km to the NW. This report presents the results of all three monitoring events. (PDF contains 37 pages.)", + "language": "en", + "place": "Silver Spring, MD", + "publisher": "NOAA/National Ocean Service/National Marine Sanctuary Program", + "series": "Marine Sanctuaries Conservation Series", + "url": "http://hdl.handle.net/1834/20117", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "Acropora palmata" + }, + { + "tag": "Coral" + }, + { + "tag": "Ecology" + }, + { + "tag": "Environment" + }, + { + "tag": "Florida Keys National Marine Sanctuary" + }, + { + "tag": "Grounding" + }, + { + "tag": "Hurricane Charley" + }, + { + "tag": "Hurricane Katrina" + }, + { + "tag": "Management" + }, + { + "tag": "Monitoring" + }, + { + "tag": "Restoration" + } + ], + "notes": [], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/Datacite JSON.js b/Datacite JSON.js index d2602037bce..508230703bf 100644 --- a/Datacite JSON.js +++ b/Datacite JSON.js @@ -8,7 +8,7 @@ "priority": 100, "inRepository": true, "translatorType": 1, - "lastUpdated": "2022-08-25 10:25:41" + "lastUpdated": "2023-07-06 12:34:43" } /* @@ -35,19 +35,25 @@ */ +const datasetType = ZU.fieldIsValidForType('title', 'dataset') + ? 'dataset' + : 'document'; + // copied from CSL JSON function parseInput() { var str, json = ""; - // Read in the whole file at once, since we can't easily parse a JSON stream. The + // Read in the whole file at once, since we can't easily parse a JSON stream. The // chunk size here is pretty arbitrary, although larger chunk sizes may be marginally // faster. We set it to 1MB. while ((str = Z.read(1048576)) !== false) json += str; try { return JSON.parse(json); - } catch(e) { + } + catch (e) { Zotero.debug(e); + return false; } } @@ -59,37 +65,41 @@ function detectImport() { return false; } - +/* eslint-disable camelcase*/ var mappingTypes = { - "book": "book", - "chapter": "bookSection", + book: "book", + chapter: "bookSection", "article-journal": "journalArticle", "article-magazine": "magazineArticle", "article-newspaper": "newspaperArticle", - "thesis": "thesis", + thesis: "thesis", "entry-encyclopedia": "encyclopediaArticle", "entry-dictionary": "dictionaryEntry", "paper-conference": "conferencePaper", - "personal_communication": "letter", - "manuscript": "manuscript", - "interview": "interview", - "motion_picture": "film", - "graphic": "artwork", - "webpage": "webpage", - "report": "report", - "bill": "bill", - "legal_case": "case", - "patent": "patent", - "legislation": "statute", - "map": "map", + personal_communication: "letter", + manuscript: "manuscript", + interview: "interview", + motion_picture: "film", + graphic: "artwork", + webpage: "webpage", + report: "report", + bill: "bill", + legal_case: "case", + patent: "patent", + legislation: "statute", + map: "map", "post-weblog": "blogPost", - "post": "forumPost", - "song": "audioRecording", - "speech": "presentation", - "broadcast": "radioBroadcast", - "dataset": "document" + post: "forumPost", + song: "audioRecording", + speech: "presentation", + broadcast: "radioBroadcast", + dataset: "dataset" }; - +/* eslint-enable camelcase*/ +// pre-6.0.26 releases don't have a dataset item type +if (datasetType == "document") { + mappingTypes.dataset = 'document'; +} function doImport() { @@ -104,7 +114,7 @@ function doImport() { } var item = new Zotero.Item(type); - if (data.types.citeproc == "dataset") { + if (data.types.citeproc == "dataset" && datasetType == "document") { item.extra = "Type: dataset"; } var title = ""; @@ -114,27 +124,27 @@ function doImport() { } if (!titleElement.titleType) { title = titleElement.title + title; - } else if (titleElement.titleType.toLowerCase() == "subtitle") { - title = title + ": " + titleElement["title"]; } - + else if (titleElement.titleType.toLowerCase() == "subtitle") { + title = title + ": " + titleElement.title; + } } item.title = title; if (data.creators) { for (let creator of data.creators) { - if (creator.nameType == "Personal") { - if (creator.familyName && creator.givenName) { - item.creators.push({ - "lastName": creator.familyName, - "firstName": creator.givenName, - "creatorType": "author" - }); - } else { - item.creators.push(ZU.cleanAuthor(creator.name, "author", true)); - } - } else { - item.creators.push({"lastName": creator.name, "creatorType": "author", "fieldMode": true}); + if (creator.familyName && creator.givenName) { + item.creators.push({ + lastName: creator.familyName, + firstName: creator.givenName, + creatorType: "author" + }); + } + else if (creator.nameType == "Personal") { + item.creators.push(ZU.cleanAuthor(creator.name, "author", true)); + } + else { + item.creators.push({ lastName: creator.name, creatorType: "author", fieldMode: 1 }); } } } @@ -142,7 +152,7 @@ function doImport() { for (let contributor of data.contributors) { let role = "contributor"; if (contributor.contributorRole) { - switch(contributor.contributorRole.toLowerCase()) { + switch (contributor.contributorRole.toLowerCase()) { case "editor": role = "editor"; break; @@ -153,18 +163,18 @@ function doImport() { // use the already assigned value } } - if (contributor.nameType == "Personal") { - if (contributor.familyName && contributor.givenName) { - item.creators.push({ - "lastName": contributor.familyName, - "firstName": contributor.givenName, - "creatorType": role - }); - } else { - item.creators.push(ZU.cleanAuthor(contributor.name, role)); - } - } else { - item.creators.push({"lastName": contributor.name, "creatorType": role, "fieldMode": true}); + if (contributor.familyName && contributor.givenName) { + item.creators.push({ + lastName: contributor.familyName, + firstName: contributor.givenName, + creatorType: role + }); + } + else if (contributor.nameType == "Personal") { + item.creators.push(ZU.cleanAuthor(contributor.name, role)); + } + else { + item.creators.push({ lastName: contributor.name, creatorType: role, fieldMode: 1 }); } } } @@ -176,15 +186,16 @@ function doImport() { for (let date of data.dates) { dates[date.dateType] = date.date; } - item.date = dates["Issued"] || dates["Updated"] || dates["Available"] || dates["Accepted"] || dates["Submitted"] || dates["Created"] || data.publicationYear; + item.date = dates.Issued || dates.Updated || dates.Available || dates.Accepted || dates.Submitted || dates.Created || data.publicationYear; } item.DOI = data.doi; //add DOI to extra for unsupported items if (item.DOI && !ZU.fieldIsValidForType("DOI", item.itemType)) { - if (item.extra){ + if (item.extra) { item.extra += "\nDOI: " + item.DOI; - } else { + } + else { item.extra = "DOI: " + item.DOI; } } @@ -211,13 +222,14 @@ function doImport() { for (let description of data.descriptions) { if (description.descriptionType == "Abstract") { item.abstractNote = description.description; - } else { + } + else { descriptionNote += "

    " + description.descriptionType + "

    \n" + description.description; } } } if (descriptionNote !== "") { - item.notes.push({"note": descriptionNote}); + item.notes.push({ note: descriptionNote }); } if (data.container) { if (data.container.type == "Series") { @@ -423,7 +435,7 @@ var testCases = [ "input": "{\n \"id\": \"https://doi.org/10.17171/2-3-12-1\",\n \"doi\": \"10.17171/2-3-12-1\",\n \"url\": \"http://repository.edition-topoi.org/collection/MAGN/single/0012/0\",\n \"types\": {\n \"resourceTypeGeneral\": \"Dataset\",\n \"resourceType\": \"3D Data\",\n \"schemaOrg\": \"Dataset\",\n \"citeproc\": \"dataset\",\n \"bibtex\": \"misc\",\n \"ris\": \"DATA\"\n },\n \"creators\": [\n {\n \"nameType\": \"Personal\",\n \"name\": \"Fritsch, Bernhard\",\n \"givenName\": \"Bernhard\",\n \"familyName\": \"Fritsch\"\n }\n ],\n \"titles\": [\n {\n \"title\": \"3D model of object V 1.2-71\"\n },\n {\n \"title\": \"Structured-light Scan, Staatliche Museen zu Berlin - Antikensammlung\",\n \"titleType\": \"Subtitle\"\n }\n ],\n \"publisher\": \"Edition Topoi\",\n \"container\": {\n \"type\": \"DataRepository\",\n \"identifier\": \"10.17171/2-3-1\",\n \"identifierType\": \"DOI\",\n \"title\": \"Architectural Fragments from Magnesia on the Maeander\"\n },\n \"subjects\": [\n {\n \"subject\": \"101 Ancient Cultures\"\n },\n {\n \"subject\": \"410-01 Building and Construction History\"\n }\n ],\n \"contributors\": [\n\n ],\n \"dates\": [\n {\n \"date\": \"2016\",\n \"dateType\": \"Updated\"\n },\n {\n \"date\": \"2016\",\n \"dateType\": \"Issued\"\n }\n ],\n \"publicationYear\": \"2016\",\n \"identifiers\": [\n {\n \"identifierType\": \"DOI\",\n \"identifier\": \"https://doi.org/10.17171/2-3-12-1\"\n }\n ],\n \"sizes\": [\n\n ],\n \"formats\": [\n \"nxs\"\n ],\n \"rightsList\": [\n\n ],\n \"descriptions\": [\n {\n \"description\": \"Architectural Fragments from Magnesia on the Maeander\",\n \"descriptionType\": \"SeriesInformation\"\n }\n ],\n \"geoLocations\": [\n\n ],\n \"fundingReferences\": [\n\n ],\n \"relatedIdentifiers\": [\n {\n \"relatedIdentifier\": \"10.17171/2-3-1\",\n \"relatedIdentifierType\": \"DOI\",\n \"relationType\": \"IsPartOf\"\n },\n {\n \"relatedIdentifier\": \"10.17171/2-3\",\n \"relatedIdentifierType\": \"DOI\",\n \"relationType\": \"IsPartOf\"\n }\n ],\n \"schemaVersion\": \"http://datacite.org/schema/kernel-3\",\n \"providerId\": \"tib\",\n \"clientId\": \"tib.topoi\",\n \"agency\": \"DataCite\",\n \"state\": \"findable\"\n}", "items": [ { - "itemType": "document", + "itemType": "dataset", "title": "3D model of object V 1.2-71: Structured-light Scan, Staatliche Museen zu Berlin - Antikensammlung", "creators": [ { @@ -433,8 +445,9 @@ var testCases = [ } ], "date": "2016", - "extra": "Type: dataset\nDOI: 10.17171/2-3-12-1", - "publisher": "Edition Topoi", + "DOI": "10.17171/2-3-12-1", + "format": "nxs", + "repository": "Edition Topoi", "url": "http://repository.edition-topoi.org/collection/MAGN/single/0012/0", "attachments": [], "tags": [ diff --git a/Duke University Press Books.js b/Duke University Press Books.js index 27165fdbe4d..cce82f4c214 100644 --- a/Duke University Press Books.js +++ b/Duke University Press Books.js @@ -9,30 +9,30 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-10-05 02:33:49" + "lastUpdated": "2023-10-23 09:05:55" } /* - ***** BEGIN LICENSE BLOCK ***** + ***** BEGIN LICENSE BLOCK ***** - Copyright © 2022 Sebastian Karcher + Copyright © 2022 Sebastian Karcher - This file is part of Zotero. + This file is part of Zotero. - Zotero is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. - Zotero is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. - You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . - ***** END LICENSE BLOCK ***** + ***** END LICENSE BLOCK ***** */ @@ -64,11 +64,9 @@ function getSearchResults(doc, checkOnly) { async function doWeb(doc, url) { if (detectWeb(doc, url) == 'multiple') { let items = await Zotero.selectItems(getSearchResults(doc, false)); - if (items) { - await Promise.all( - Object.keys(items) - .map(url => requestDocument(url).then(scrape)) - ); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } } else { @@ -213,6 +211,40 @@ var testCases = [ "type": "web", "url": "https://www.dukeupress.edu/books/browse?sortid=7", "items": "multiple" + }, + { + "type": "web", + "url": "https://www.dukeupress.edu/beyond-this-narrow-now", + "items": [ + { + "itemType": "book", + "title": "\"Beyond This Narrow Now\": Or, Delimitations, of W. E. B. Du Bois", + "creators": [ + { + "firstName": "Nahum Dimitri", + "lastName": "Chandler", + "creatorType": "author" + } + ], + "date": "2022", + "ISBN": "9781478014805", + "abstractNote": "In “Beyond This Narrow Now” Nahum Dimitri Chandler shows that the premises of W. E. B. Du Bois's thinking at the turn of the twentieth century stand as fundamental references for the whole itinerary of his thought. Opening with a distinct approach to the legacy of Du Bois, Chandler proceeds through a series of close readings of Du Bois's early essays, previously unpublished or seldom studied, with discrete annotations of The Souls of Black Folk: Essays and Sketches of 1903, elucidating and elaborating basic epistemological terms of his thought. With theoretical attention to how the African American stands as an example of possibility for Du Bois and renders problematic traditional ontological thought, Chandler also proposes that Du Bois's most well-known phrase—“the problem of the color line”—sustains more conceptual depth than has yet been understood, with pertinence for our accounts of modern systems of enslavement and imperial colonialism and the incipient moments of modern capitalization. Chandler's work exemplifies a more profound engagement with Du Bois, demonstrating that he must be re-read, appreciated, and studied anew as a philosophical writer and thinker contemporary to our time.", + "libraryCatalog": "Duke University Press Books", + "numPages": "328", + "place": "Durham, NC", + "publisher": "Duke University Press", + "shortTitle": "\"Beyond This Narrow Now\"", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/E-periodica Switzerland.js b/E-periodica Switzerland.js new file mode 100644 index 00000000000..03763d6293e --- /dev/null +++ b/E-periodica Switzerland.js @@ -0,0 +1,347 @@ +{ + "translatorID": "dbfd99e3-6925-4b71-92b8-12b02aa875fc", + "label": "E-periodica Switzerland", + "creator": "Alain Borel", + "target": "^https?://(www|news?)\\.e-periodica\\.ch", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-08-15 20:15:50" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Alain Borel + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { + if (url.includes('/digbib/view')) { + return "journalArticle"; + } + else if (getSearchResults(doc, true)) { + return "multiple"; + } + else { + return false; + } +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + // Zotero.debug(items); + var found = false; + var rows = doc.querySelectorAll('h2.ep-result__title > a'); + for (let row of rows) { + //Zotero.debug(row.textContent); + let href = row.href; + //Zotero.debug(href); + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + // Zotero.debug(items[href]); + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let resultUrl of Object.keys(items)) { + await scrape(resultUrl); + } + } + else { + // The journalArticle type will be applicable in general unless we find multiple refs. + await scrape(url); + } +} + +async function scrape(url) { + // In general the article ID is given the pid parameter in the URL + // If the URL ends with a hash/fragment identifier, + // the final digits of the pid parameter (after a double colon) must be replaced with the hash ID + // e.g. alp-001:1907:2::332#375 => alp-001:1907:2::375 + let articleURL = new URL(url); + let articleID = articleURL.searchParams.get("pid"); + let articleViewFragment = articleURL.hash.replace(/^#/, ""); // trim leading # + if (/\d+/.test(articleViewFragment)) { + // Normalize article ID by replacing the last segment with the real + // page id if any + articleID = articleID.replace(/::\d+$/, "::" + articleViewFragment); + } + let pageinfoUrl = "https://www.e-periodica.ch/digbib/ajax/pageinfo?pid=" + encodeURIComponent(articleID); + + //Zotero.debug('JSON URL ' + pageinfoUrl); + let epJSON = await requestJSON(pageinfoUrl); + //Zotero.debug(epJSON); + let risURL; + if (epJSON.articles.length == 0) { + // Fallback for non-article content, listed as Werbung, Sonstiges and various others: + // this information is unfortunately not included in the JSON metadata => let's add a reasonable pseudo-title + epJSON.articles = [{ title: "Untitled" }]; + } + if (epJSON.articles["0"].hasRisLink) { + risURL = '/view/' + epJSON.articles["0"].risLink; + } + + // Zotero.debug(risURL); + var pdfURL = null; + if (epJSON.articles["0"].hasPdfLink) { + pdfURL = epJSON.articles["0"].pdfLink; + } + // Zotero.debug(pdfURL); + if (risURL) { + let risText = await requestText(risURL); + processRIS(risText, pdfURL); + } + else { + var item = new Zotero.Item("journalArticle"); + item.title = epJSON.articles["0"].title.replace(' : ', ': '); + item.publicationTitle = epJSON.journalTitle.replace(' : ', ': '); + var numyear = epJSON.volumeNumYear.split(/[ ()]/).filter(element => element); + if (numyear.length > 1) { + item.date = numyear.slice(-1); + } + if (numyear.length > 0) { + item.volume = numyear[0]; + } + if (epJSON.issueNumber) { + item.issue = epJSON.issueNumber; + } + if (epJSON.viewerLink.length > 0) { + if (epJSON.viewerLink.indexOf("http") == 0) { + item.url = epJSON.viewerLink; + } + else { + item.url = "https://www.e-periodica.ch" + epJSON.viewerLink; + } + } + if (epJSON.pdfLink) { + if (epJSON.pdfLink.indexOf("http") == 0) { + pdfURL = epJSON.pdfLink; + } + else { + pdfURL = "https://www.e-periodica.ch" + epJSON.pdfLink; + } + } + if (pdfURL) { + item.attachments.push({ + url: pdfURL, + title: "Full Text PDF", + type: "application/pdf" + }); + } + item.complete(); + } +} + +function processRIS(risText, pdfURL) { + // load translator for RIS + var translator = Zotero.loadTranslator("import"); + translator.setTranslator("32d59d2d-b65a-4da4-b0a3-bdd3cfb979e7"); + // Z.debug(text); + + translator.setString(risText); + translator.setHandler("itemDone", function (obj, item) { + // Don't save HTML snapshot from 'UR' tag + item.attachments = []; + + // change colon spacing in title and publicationTitle + item.title = item.title.replace(' : ', ': '); + + if (item.publicationTitle) { + item.publicationTitle = item.publicationTitle.replace(' : ', ': '); + } + + if (item.title == item.title.toUpperCase()) { + item.title = ZU.capitalizeTitle(item.title.toLowerCase(), true); + } + + // Retrieve fulltext + if (pdfURL !== null) { + item.attachments.push({ + url: pdfURL, + title: "Full Text PDF", + type: "application/pdf" + }); + } + + // DB in RIS maps to archive; we don't want that + delete item.archive; + + item.complete(); + }); + + translator.getTranslatorObject(function (trans) { + trans.doImport(); + }); +}/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://www.e-periodica.ch/digbib/view?pid=enh-006%3A2018%3A11::121#133", + "detectedItemType": "journalArticle", + "items": [ + { + "itemType": "journalArticle", + "title": "Untersuchungen zur aktuellen Verbreitung der schweizerischen Laufkäfer (Coleoptera: Carabidae): Zwischenbilanz", + "creators": [ + { + "lastName": "Hoess", + "firstName": "René", + "creatorType": "author" + }, + { + "lastName": "Chittaro", + "firstName": "Yannick", + "creatorType": "author" + }, + { + "lastName": "Walter", + "firstName": "Thomas", + "creatorType": "author" + } + ], + "date": "2018", + "DOI": "10.5169/seals-986030", + "ISSN": "1662-8500", + "libraryCatalog": "E-periodica Switzerland", + "pages": "129", + "publicationTitle": "Entomo Helvetica: entomologische Zeitschrift der Schweiz", + "shortTitle": "Untersuchungen zur aktuellen Verbreitung der schweizerischen Laufkäfer (Coleoptera", + "volume": "11", + "attachments": [ + { + "title": "Full Text PDF", + "type": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.e-periodica.ch/digbib/view?pid=bts-004%3A2011%3A137%3A%3A254&referrer=search#251", + "detectedItemType": "journalArticle", + "items": [ + { + "itemType": "journalArticle", + "title": "Décentralisation, opportunités et contraintes", + "creators": [ + { + "lastName": "Fignolé", + "firstName": "Jean-Claude", + "creatorType": "author" + } + ], + "date": "2011", + "DOI": "10.5169/seals-144646", + "ISSN": "0251-0979", + "issue": "05-06", + "libraryCatalog": "E-periodica Switzerland", + "pages": "14", + "publicationTitle": "Tracés: bulletin technique de la Suisse romande", + "volume": "137", + "attachments": [ + { + "title": "Full Text PDF", + "type": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.e-periodica.ch/digbib/view?pid=alp-001%3A1907%3A2%3A%3A332#375", + "detectedItemType": "journalArticle", + "items": [ + { + "itemType": "journalArticle", + "title": "Stimmen und Meinungen: schweizerisches Nationaldrama?", + "creators": [ + { + "lastName": "Falke", + "firstName": "Konrad", + "creatorType": "author" + } + ], + "date": "1907-1908", + "DOI": "10.5169/seals-747870", + "issue": "12", + "libraryCatalog": "E-periodica Switzerland", + "pages": "364", + "publicationTitle": "Berner Rundschau: Halbmonatsschrift für Dichtung, Theater, Musik und bildende Kunst in der Schweiz", + "shortTitle": "Stimmen und Meinungen", + "volume": "2", + "attachments": [ + { + "title": "Full Text PDF", + "type": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.e-periodica.ch/digbib/view?pid=bts-004%3A2011%3A137%3A%3A262#262", + "detectedItemType": "journalArticle", + "items": [ + { + "itemType": "journalArticle", + "title": "Untitled", + "creators": [], + "date": "2011", + "issue": "05-06", + "libraryCatalog": "E-periodica Switzerland", + "publicationTitle": "Tracés: bulletin technique de la Suisse romande", + "url": "https://www.e-periodica.ch/digbib/view?pid=bts-004%3A2011%3A137%3A%3A262", + "volume": "137", + "attachments": [ + { + "title": "Full Text PDF", + "type": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/EBSCO Discovery Layer.js b/EBSCO Discovery Layer.js index 73c05a9375f..579c60763f6 100644 --- a/EBSCO Discovery Layer.js +++ b/EBSCO Discovery Layer.js @@ -2,14 +2,14 @@ "translatorID": "660fcf3e-3414-41b8-97a5-e672fc2e491d", "label": "EBSCO Discovery Layer", "creator": "Sebastian Karcher", - "target": "^https?://discovery\\.ebsco\\.com/", + "target": "^https?://(discovery|research)\\.ebsco\\.com/", "minVersion": "5.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-02-12 03:36:07" + "lastUpdated": "2023-11-27 04:12:52" } /* @@ -35,7 +35,7 @@ ***** END LICENSE BLOCK ***** */ -const itemRegex = /\/c\/([^/]+)\/(?:details|viewer\/pdf)\/([^?]+)/; +const itemRegex = /\/c\/([^/]+)(?:\/search)?\/(?:details|viewer\/pdf)\/([^?]+)/; function detectWeb(doc, url) { if (itemRegex.test(url)) { if (url.includes("/viewer/pdf")) { diff --git a/ERIC.js b/ERIC.js index aed4ae5de9f..7ab177cc3e7 100644 --- a/ERIC.js +++ b/ERIC.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 12, "browserSupport": "gcsibv", - "lastUpdated": "2022-11-09 21:08:42" + "lastUpdated": "2023-08-22 04:49:47" } /* @@ -89,11 +89,9 @@ function getSearchResults(doc, checkOnly) { async function doWeb(doc, url) { if (detectWeb(doc, url) == 'multiple') { let items = await Zotero.selectItems(getSearchResults(doc, false)); - if (items) { - await Promise.all( - Object.keys(items) - .map(url => requestDocument(url).then(scrape)) - ); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } } else { diff --git a/Electronic Colloquium on Computational Complexity.js b/Electronic Colloquium on Computational Complexity.js index 43fe337d059..3191947f5fc 100644 --- a/Electronic Colloquium on Computational Complexity.js +++ b/Electronic Colloquium on Computational Complexity.js @@ -1,116 +1,120 @@ { "translatorID": "09a9599e-c20e-a405-d10d-35ad4130a426", "label": "Electronic Colloquium on Computational Complexity", - "creator": "Jonas Schrieb", - "target": "^https?://eccc\\.weizmann\\.ac\\.il/", + "creator": "Jonas Schrieb and Morgan Shirley", + "target": "^https?://eccc\\.weizmann\\.ac\\.il/(title|year|keyword|report|search)", "minVersion": "1.0.0b3.r1", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, - "browserSupport": "gcsib", - "lastUpdated": "2017-01-05 17:11:41" + "browserSupport": "gcsibv", + "lastUpdated": "2023-08-14 06:03:23" } -function detectWeb(doc, url) { - var singleRe = /^https?:\/\/eccc\.weizmann\.ac\.il\/report\/\d{4}\/\d{3}/; - var multipleRe = /^https?:\/\/eccc\.weizmann\.ac\.il\/(title|year|keyword)\//; - if (singleRe.test(url)) { - return "report"; - } else if (multipleRe.test(url)) { - return "multiple"; - } -} +/* + ***** BEGIN LICENSE BLOCK ***** -function scrape(doc) { - var newItem = new Zotero.Item("report"); + Copyright © 2023 Jonas Schrieb and Morgan Shirley - var url = doc.location.href; - var tmp = url.match(/\/(\d{4})\/(\d{3})\/$/); - newItem.date = tmp[1]; - newItem.reportNumber = tmp[2]; - newItem.url = url; - + This file is part of Zotero. + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. - var titleXPath = "id('box')//h4"; - newItem.title = doc.evaluate(titleXPath, doc, null, XPathResult.ANY_TYPE, null).iterateNext().textContent; + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + ***** END LICENSE BLOCK ***** +*/ - var authorsXPath = "id('box')//a[contains(@href,'author')]"; - var authors = doc.evaluate(authorsXPath, doc, null, XPathResult.ANY_TYPE, null); - var nextAuthor; - while (nextAuthor = authors.iterateNext()) { - newItem.creators.push(Zotero.Utilities.cleanAuthor(nextAuthor.textContent, "author")); - } +const preprintType = ZU.fieldIsValidForType('title', 'preprint') + ? 'preprint' + : 'report'; - - - var keywordsXPath = "id('box')//a[contains(@href,'keyword')]"; - var keywords = doc.evaluate(keywordsXPath, doc, null, XPathResult.ANY_TYPE, null); - var nextKeyword; - var i = 0; - while (nextKeyword = keywords.iterateNext()) { - newItem.tags[i++] = nextKeyword.textContent; +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('a[href^="/report/"]'); + for (let row of rows) { + let href = row.href; + let title = text(row, "h4"); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; } + return found ? items : false; +} - - - var abstractXPath = "id('box')/text()"; - var abstractLines = doc.evaluate(abstractXPath, doc, null, XPathResult.ANY_TYPE, null); - newItem.abstractNote = ""; - var nextLine; - while (nextLine = abstractLines.iterateNext()) { - newItem.abstractNote += nextLine.textContent; +function detectWeb(doc, url) { + var multipleRe = /^https?:\/\/eccc\.weizmann\.ac\.il\/(title|year|keyword|search)\//; + var singleRe = /^https?:\/\/eccc\.weizmann\.ac\.il\/report\//; + if (multipleRe.test(url)) { + return getSearchResults(doc, true) && "multiple"; + } + else if (singleRe.test(url)) { + return preprintType; } + else return false; +} +async function scrape(doc, url = doc.location.href) { + let translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + + translator.setHandler('itemDone', (_obj, item) => { + item.publisher = "Electronic Colloquium on Computational Complexity"; + // Keywords and abstract are not in the metadata; scrape from webpage + var keywords = doc.querySelectorAll("#box a[href^='/keyword/']"); + for (let i = 0; i < keywords.length; i++) { + item.tags[i] = keywords[i].textContent; + } - newItem.attachments = [ - {url:url, title:"ECCC Snapshot", mimeType:"text/html"}, - {url:url+"download", title:"ECCC Full Text PDF", mimeType:"application/pdf"} - ]; + var abstractParagraphs = doc.querySelectorAll("#box p"); + item.abstractNote = ""; + for (let i = 0; i < abstractParagraphs.length; i++) { + item.abstractNote += abstractParagraphs[i].innerText + "\n"; + } + item.complete(); + }); - newItem.complete(); + let em = await translator.getTranslatorObject(); + em.itemType = preprintType; + await em.doWeb(doc, url); } -function doWeb(doc, url) { - var articles = new Array(); - var items = new Object(); - var nextTitle; - - if (detectWeb(doc, url) == "multiple") { - var titleXPath = "//a[starts-with(@href,'/report/')]/h4"; - var linkXPath = "//a[starts-with(@href,'/report/')][h4]"; - - var titles = doc.evaluate(titleXPath, doc, null, XPathResult.ANY_TYPE, null); - var links = doc.evaluate(linkXPath, doc, null, XPathResult.ANY_TYPE, null); - while (nextTitle = titles.iterateNext()) { - nextLink = links.iterateNext(); - items[nextLink.href] = nextTitle.textContent; +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } - Zotero.selectItems(items, function (items) { - if (!items) { - Zotero.done(); - } - for (var i in items) { - articles.push(i); - } - ZU.processDocuments(articles, scrape); - }); - } else { - scrape(doc, url) } -} -/** BEGIN TEST CASES **/ + else { + await scrape(doc, url); + } +}/** BEGIN TEST CASES **/ var testCases = [ { "type": "web", "url": "https://eccc.weizmann.ac.il/report/2006/067/", + "detectedItemType": "preprint", "items": [ { - "itemType": "report", + "itemType": "preprint", + "title": "On the Impact of Combinatorial Structure on Congestion Games", "creators": [ { "firstName": "Heiner", @@ -128,39 +132,98 @@ var testCases = [ "creatorType": "author" } ], - "notes": [], + "date": "2006/5/28", + "abstractNote": "We study the impact of combinatorial structure in congestion games on the complexity of computing pure Nash equilibria and the convergence time of best response sequences. In particular, we investigate which properties of the strategy spaces of individual players ensure a polynomial convergence time. We show, if the strategy space of each player consists of the bases of a matroid over the set of resources, then the lengths of all best response sequences are polynomially bounded in the number of players and resources. We can also prove that this result is tight, that is, the matroid property is a necessary and sufficient condition on the players' strategy spaces for guaranteeing polynomial time convergence to a Nash equilibrium. In addition, we present an approach that enables us to devise hardness proofs for various kinds of combinatorial games, including first results about the hardness of market sharing games and congestion games for overlay network design. Our approach also yields a short proof for the PLS-completeness of network congestion games.", + "archiveID": "TR06-067", + "language": "en", + "libraryCatalog": "eccc.weizmann.ac.il", + "repository": "Electronic Colloquium on Computational Complexity", + "url": "https://eccc.weizmann.ac.il/report/2006/067/", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], "tags": [ - "Combinatorial Structure", - "Congestion Games", - "Convergence Time", - "PLS-Completeness" + { + "tag": "Combinatorial Structure" + }, + { + "tag": "Congestion Games" + }, + { + "tag": "Convergence Time" + }, + { + "tag": "PLS-Completeness" + } ], - "seeAlso": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://eccc.weizmann.ac.il/report/2007/112/", + "detectedItemType": "preprint", + "items": [ + { + "itemType": "preprint", + "title": "Unbounded-Error Communication Complexity of Symmetric Functions", + "creators": [ + { + "firstName": "Alexander A.", + "lastName": "Sherstov", + "creatorType": "author" + } + ], + "date": "2007/11/12", + "abstractNote": "The sign-rank of a real matrix M is the least rank\nof a matrix R in which every entry has the same sign as the\ncorresponding entry of M. We determine the sign-rank of every\nmatrix of the form M=[ D(|x AND y|) ]_{x,y}, where\nD:{0,1,...,n}->{-1,+1} is given and x and y range over {0,1}^n.\nSpecifically, we prove that the sign-rank of M equals\n2^{\\tilde Theta(k)}, where k is the number of times D changes\nsign in {0,1,...,n}.\nPut differently, we prove an optimal lower bound\non the unbounded-error communication complexity of every\nsymmetric function, i.e., a function of the form\nf(x,y)=D(|x AND y|) for some D. The unbounded-error model is\nessentially the most powerful of all models of communication\n(both classical and quantum), and proving lower bounds in it\nis a substantial challenge. The only previous nontrivial lower\nbounds for this model appear in the groundbreaking work of\nForster (2001) and its extensions. As corollaries to our\nresult, we give new lower bounds for PAC learning and for\nthreshold-of-majority circuits.\nThe technical content of our proof is diverse and\nfeatures random walks on (Z_2)^n, discrete approximation theory,\nthe Fourier transform on (Z_2)^n, linear-programming duality,\nand matrix analysis.", + "archiveID": "TR07-112", + "language": "en", + "libraryCatalog": "eccc.weizmann.ac.il", + "repository": "Electronic Colloquium on Computational Complexity", + "url": "https://eccc.weizmann.ac.il/report/2007/112/", "attachments": [ { - "url": "https://eccc.weizmann.ac.il/report/2006/067/", - "title": "ECCC Snapshot", - "mimeType": "text/html" + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "Communication complexity" }, { - "url": "https://eccc.weizmann.ac.il/report/2006/067/download", - "title": "ECCC Full Text PDF", - "mimeType": "application/pdf" + "tag": "Sign-rank" + }, + { + "tag": "Unbounded-error communication complexity" } ], - "date": "2006", - "reportNumber": "067", - "url": "https://eccc.weizmann.ac.il/report/2006/067/", - "title": "On the Impact of Combinatorial Structure on Congestion Games", - "abstractNote": "", - "libraryCatalog": "Electronic Colloquium on Computational Complexity", - "accessDate": "CURRENT_TIMESTAMP" + "notes": [], + "seeAlso": [] } ] }, { "type": "web", - "url": "https://eccc.weizmann.ac.il/keyword/13486/", + "url": "https://eccc.weizmann.ac.il/search/?search=combinatorial", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://eccc.weizmann.ac.il/search/?search=asdf", + "detectedItemType": false, + "items": [] + }, + { + "type": "web", + "url": "https://eccc.weizmann.ac.il/keyword/14114/", + "detectedItemType": "multiple", "items": "multiple" } ] diff --git a/Embedded Metadata.js b/Embedded Metadata.js index 5b7f264aac9..36d6fd3bba1 100644 --- a/Embedded Metadata.js +++ b/Embedded Metadata.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-04-24 14:42:56" + "lastUpdated": "2023-08-16 15:18:29" } /* @@ -179,7 +179,13 @@ function getContentText(doc, name, strict, all) { } function getContent(doc, name, strict) { - var xpath = '/x:html/x:head/x:meta[' + var xpath = '/x:html/' + + ( + exports.searchForMetaTagsInBody + ? '*[local-name() = "head" or local-name() = "body"]' + : 'x:head' // default + ) + + '/x:meta[' + (strict ? '@name' : 'substring(@name, string-length(@name)-' + (name.length - 1) + ')') + '="' + name + '"]/'; return ZU.xpath(doc, xpath + '@content | ' + xpath + '@contents', namespaces); @@ -234,10 +240,14 @@ function detectWeb(doc, url) { function init(doc, url, callback, forceLoadRDF) { getPrefixes(doc); - var metaTags = doc.head.getElementsByTagName("meta"); + let metaSelector + = exports.searchForMetaTagsInBody + ? "head > meta, body > meta" + : "head > meta"; // default + var metaTags = doc.querySelectorAll(metaSelector); Z.debug("Embedded Metadata: found " + metaTags.length + " meta tags."); if (forceLoadRDF /* check if this is called from doWeb */ && !metaTags.length) { - if (doc.head) { + if (!exports.searchForMetaTagsInBody && doc.head) { Z.debug(doc.head.innerHTML .replace(/|\/>)/ig, '') .replace(/]+>/ig, '') @@ -245,7 +255,7 @@ function init(doc, url, callback, forceLoadRDF) { ); } else { - Z.debug("Embedded Metadata: No head tag"); + Z.debug("Embedded Metadata: No tag found in body or head tag"); } } @@ -736,7 +746,7 @@ function tryOgAuthors(doc) { var authors = []; var ogAuthors = ZU.xpath(doc, '//meta[@property="article:author" or @property="video:director" or @property="music:musician"]'); for (var i = 0; i < ogAuthors.length; i++) { - if (ogAuthors[i].content && /(https?:\/\/)?[\da-z.-]+\.[a-z.]{2,6}/.test(ogAuthors[i].content) && ogAuthors[i].content !== "false") { + if (ogAuthors[i].content && !/(https?:\/\/)?[\da-z.-]+\.[a-z.]{2,6}/.test(ogAuthors[i].content) && ogAuthors[i].content !== "false") { authors.push(ZU.cleanAuthor(ogAuthors[i].content, "author")); } } @@ -1001,6 +1011,9 @@ var exports = { detectWeb: detectWeb, addCustomFields: addCustomFields, itemType: false, + // workaround for meta tags in body caused by parsing invalid HTML; only + // use as a last resort + searchForMetaTagsInBody: false, // activate/deactivate splitting tags in final data cleanup when they contain commas or semicolons splitTags: true, fixSchemaURI: setPrefixRemap @@ -2008,6 +2021,31 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://www.nhs.uk/conditions/baby/babys-development/behaviour/separation-anxiety/", + "items": [ + { + "itemType": "webpage", + "title": "Separation anxiety", + "creators": [], + "date": "7 Dec 2020, 4:40 p.m.", + "abstractNote": "Separation anxiety is a normal part of your child's development. Find out how to handle the times when your baby or toddler cries or is clingy when you leave them.", + "language": "en", + "url": "https://www.nhs.uk/conditions/baby/babys-development/behaviour/separation-anxiety/", + "websiteTitle": "nhs.uk", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/Encyclopedia of Korean Culture.js b/Encyclopedia of Korean Culture.js new file mode 100644 index 00000000000..545928467de --- /dev/null +++ b/Encyclopedia of Korean Culture.js @@ -0,0 +1,221 @@ +{ + "translatorID": "dc879929-ae39-45b3-b49b-dab2c80815ab", + "label": "Encyclopedia of Korean Culture", + "creator": "jacoblee36251", + "target": "^https?://(www\\.)?encykorea\\.aks\\.ac\\.kr/Article/", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-09-15 20:07:33" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 jacoblee36251 + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + + +function detectWeb(doc, url) { + if (/^https?:\/\/[^/]+\/Article\/E\d+/.test(url)) { + return 'encyclopediaArticle'; + } + else if (getSearchResults(doc, true)) { + return 'multiple'; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('li.item > a[href^="/Article/E"]'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.querySelector('div.title').textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrape(doc, url = doc.location.href) { + var item = new Zotero.Item('encyclopediaArticle'); + + item.title = ZU.trimInternal(text(doc, ".content-head-title")); + item.encyclopediaTitle = "한국민족문화대백과사전 [Encyclopedia of Korean Culture]"; + item.publisher = "Academy of Korean Studies"; + item.language = "ko"; + // Clean url by removing # terms + item.url = url.replace(/#.*/, ""); + + // Author processing; may be 0 or more names, would be in Korean + var authors = doc.querySelector('div.author-wrap > span'); + + if (authors) { + authors = authors.textContent; + // For simplicity, assume one character surnames for everybody (there are rare exceptions) + for (let author of authors.split('·')) { + item.creators.push({ + lastName: author[0], + firstName: author.slice(1), + creatorType: "author" + }); + } + } + item.attachments.push({ title: "Snapshot", document: doc, mimeType: "text/html" }); + item.complete(); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://encykorea.aks.ac.kr/Article/E0013414#cm_multimedia", + "detectedItemType": "encyclopediaArticle", + "items": [ + { + "itemType": "encyclopediaArticle", + "title": "다대포 (多大浦)", + "creators": [ + { + "lastName": "오", + "firstName": "건환", + "creatorType": "author" + }, + { + "lastName": "김", + "firstName": "건유", + "creatorType": "author" + } + ], + "encyclopediaTitle": "한국민족문화대백과사전 [Encyclopedia of Korean Culture]", + "language": "ko", + "libraryCatalog": "Encyclopedia of Korean Culture", + "publisher": "Academy of Korean Studies", + "url": "https://encykorea.aks.ac.kr/Article/E0013414", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://encykorea.aks.ac.kr/Article/E0002855", + "detectedItemType": "encyclopediaArticle", + "items": [ + { + "itemType": "encyclopediaArticle", + "title": "경주 남산 탑곡 마애불상군 (慶州 南山 塔谷 磨崖佛像群)", + "creators": [], + "encyclopediaTitle": "한국민족문화대백과사전 [Encyclopedia of Korean Culture]", + "language": "ko", + "libraryCatalog": "Encyclopedia of Korean Culture", + "publisher": "Academy of Korean Studies", + "url": "https://encykorea.aks.ac.kr/Article/E0002855", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://encykorea.aks.ac.kr/Article/E0025488", + "detectedItemType": "encyclopediaArticle", + "items": [ + { + "itemType": "encyclopediaArticle", + "title": "사랑 (舍廊)", + "creators": [ + { + "lastName": "김", + "firstName": "동욱", + "creatorType": "author" + } + ], + "encyclopediaTitle": "한국민족문화대백과사전 [Encyclopedia of Korean Culture]", + "language": "ko", + "libraryCatalog": "Encyclopedia of Korean Culture", + "publisher": "Academy of Korean Studies", + "url": "https://encykorea.aks.ac.kr/Article/E0025488", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://encykorea.aks.ac.kr/Article/Search/%ED%95%99%EC%9B%90?field=&type=&alias=false&body=false&containdesc=false&keyword=%ED%95%99%EC%9B%90", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://encykorea.aks.ac.kr/Article/List/Field/%EC%98%88%EC%88%A0%C2%B7%EC%B2%B4%EC%9C%A1%3E%EC%A1%B0%EA%B0%81", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://encykorea.aks.ac.kr/Article/List/Type/%EC%9C%A0%EC%A0%81", + "detectedItemType": "multiple", + "items": "multiple" + } +] +/** END TEST CASES **/ diff --git a/Foreign Affairs.js b/Foreign Affairs.js index 853b16c7832..b5c7e833952 100644 --- a/Foreign Affairs.js +++ b/Foreign Affairs.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-11-10 02:39:57" + "lastUpdated": "2023-08-23 12:37:06" } /* @@ -47,12 +47,19 @@ function detectWeb(doc, _url) { } function getSearchResults(doc, checkOnly) { + let isIssues = /^https:\/\/[^/]+\/issues\/.+/.test(doc.location.href); var items = {}; var found = false; var rows = doc.querySelectorAll('article.article-card > a, div.article-data > h2.title > a'); for (let row of rows) { let href = row.href; - let title = ZU.trimInternal(row.textContent); + let title; + if (isIssues) { + title = text(row, ".article-card-title"); + } + else { + title = ZU.trimInternal(row.textContent); + } if (!href || !title) continue; if (checkOnly) return true; found = true; @@ -64,11 +71,9 @@ function getSearchResults(doc, checkOnly) { async function doWeb(doc, url) { if (detectWeb(doc, url) == 'multiple') { let items = await Zotero.selectItems(getSearchResults(doc, false)); - if (items) { - await Promise.all( - Object.keys(items) - .map(url => requestDocument(url).then(scrape)) - ); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } } else { @@ -80,23 +85,26 @@ async function scrape(doc, url = doc.location.href) { var item = new Zotero.Item("magazineArticle"); var author = text(doc, '.article-byline-author'); var tags = doc.querySelectorAll('.article-footer--tag-item'); - var issue = text(doc, 'span.article-header--metadata-date>a'); - item.issue = issue.replace('Issue', ''); + let issueNode = doc.querySelector(".article-header--metadata-date > a"); + if (issueNode) { + var volumeTitle = ZU.trimInternal(issueNode.textContent.trim()); + // the digits are yyyy/vol/num + let issueMatch = issueNode.href.match(/\/issues\/\d+\/(\d+)\/(\d+)$/); + if (issueMatch) { + item.volume = issueMatch[1]; + item.issue = issueMatch[2]; + } + } + if (volumeTitle) { + if (!item.extra) item.extra = ""; + item.extra += `\nVolume Title: ${volumeTitle}`; + } - // the date published can be unreliable. If the issue is earlier than than the - // publication date, use the former - // e.g. https://www.foreignaffairs.com/articles/middle-east/2012-01-01/time-attack-iran item.date = attr(doc, 'meta[property="article:published_time"]', 'content'); item.title = attr(doc, 'meta[property="og:title"]', 'content'); item.abstractNote = attr(doc, 'meta[name="abstract"]', 'content'); - if (item.issue) { - var issueYear = item.issue.match(/\d{4}/); - } if (item.date) { - var dateYear = item.date.match(/\d{4}/); - } - if (issueYear && (!item.date || dateYear > issueYear)) { - item.date = issueYear[0]; + item.date = ZU.strToISO(item.date); } let authors = author.split(/, and|and |, /); @@ -121,6 +129,7 @@ var testCases = [ { "type": "web", "url": "http://www.foreignaffairs.com/issues/2012/91/01", + "defer": true, "items": "multiple" }, { @@ -137,14 +146,16 @@ var testCases = [ "creatorType": "author" } ], - "date": "2003", + "date": "2003-05-01", "ISSN": "0015-7120", "abstractNote": "A fascinating and well-translated account of Argentina's misadventures over the last century by one of that country's brightest historians. Absorbing vast amounts of British capital and tens of thousands of European immigrants, Argentina began the century with great promise. In 1914, with half of its population still foreign, a dynamic society had emerged that was both open and mobile.", - "issue": "May/June 2003", + "extra": "Volume Title: May/June 2003", + "issue": "3", "language": "en-US", "libraryCatalog": "Foreign Affairs", "publicationTitle": "Foreign Affairs", "url": "https://www.foreignaffairs.com/reviews/capsule-review/2003-05-01/history-argentina-twentieth-century", + "volume": "82", "attachments": [ { "title": "Snapshot", @@ -178,14 +189,16 @@ var testCases = [ "creatorType": "author" } ], - "date": "2012", + "date": "2012-01-01", "ISSN": "0015-7120", "abstractNote": "Opponents of military action against Iran assume a U.S. strike would be far more dangerous than simply letting Tehran build a bomb. Not so, argues this former Pentagon defense planner. With a carefully designed attack, Washington could mitigate the costs and spare the region and the world from an unacceptable threat.", - "issue": "January/February 2012", + "extra": "Volume Title: January/February 2012", + "issue": "1", "language": "en-US", "libraryCatalog": "Foreign Affairs", "publicationTitle": "Foreign Affairs", "url": "https://www.foreignaffairs.com/articles/middle-east/2012-01-01/time-attack-iran", + "volume": "91", "attachments": [ { "title": "Snapshot", @@ -208,6 +221,9 @@ var testCases = [ { "tag": "Middle East" }, + { + "tag": "North America" + }, { "tag": "Nuclear Weapons & Proliferation" }, @@ -226,6 +242,9 @@ var testCases = [ { "tag": "U.S. Foreign Policy" }, + { + "tag": "United States" + }, { "tag": "War & Military Strategy" } @@ -254,14 +273,16 @@ var testCases = [ "creatorType": "author" } ], - "date": "2014-09-04T21:23:08-04:00", + "date": "2014-08-11", "ISSN": "0015-7120", "abstractNote": "Most economists agree that the global economy is stagnating and that governments need to stimulate growth, but lowering interest rates still further could spur a damaging cycle of booms and busts. Instead, central banks should hand consumers cash directly.", - "issue": "September/October 2014", + "extra": "Volume Title: September/October 2014", + "issue": "5", "language": "en-US", "libraryCatalog": "Foreign Affairs", "publicationTitle": "Foreign Affairs", "url": "https://www.foreignaffairs.com/articles/united-states/2014-08-11/print-less-transfer-more", + "volume": "93", "attachments": [ { "title": "Snapshot", @@ -278,6 +299,9 @@ var testCases = [ { "tag": "Europe" }, + { + "tag": "North America" + }, { "tag": "United States" } @@ -301,7 +325,7 @@ var testCases = [ "creatorType": "author" } ], - "date": "2014-09-08T22:34:01-04:00", + "date": "2014-09-08", "ISSN": "0015-7120", "abstractNote": "India needs fundamental change: its rural land rights system is a mess, its manufacturing sector has been strangled by labor market restrictions, and its states are poorly integrated. But, so far, Modi has squandered major opportunities to establish his economic vision.", "language": "en-US", @@ -320,6 +344,9 @@ var testCases = [ }, { "tag": "India" + }, + { + "tag": "South Asia" } ], "notes": [], @@ -351,14 +378,16 @@ var testCases = [ "creatorType": "author" } ], - "date": "2022-09-07T21:41:06-04:00", + "date": "2022-08-31", "ISSN": "0015-7120", "abstractNote": "How AI distorts decision-making and makes dictators more dangerous.", - "issue": "September/October 2022", + "extra": "Volume Title: September/October 2022", + "issue": "5", "language": "en-US", "libraryCatalog": "Foreign Affairs", "publicationTitle": "Foreign Affairs", "url": "https://www.foreignaffairs.com/world/spirals-delusion-artificial-intelligence-decision-making", + "volume": "101", "attachments": [ { "title": "Snapshot", @@ -367,10 +396,10 @@ var testCases = [ ], "tags": [ { - "tag": "Authoritarianism" + "tag": "Artificial Intelligence" }, { - "tag": "Disinformation" + "tag": "Authoritarianism" }, { "tag": "Foreign Affairs: 100 Years" @@ -378,9 +407,18 @@ var testCases = [ { "tag": "Intelligence" }, + { + "tag": "Politics & Society" + }, + { + "tag": "Propaganda & Disinformation" + }, { "tag": "Science & Technology" }, + { + "tag": "Security" + }, { "tag": "World" } @@ -393,6 +431,7 @@ var testCases = [ { "type": "web", "url": "https://www.foreignaffairs.com/search/argentina", + "defer": true, "items": "multiple" } ] diff --git a/Google Scholar.js b/Google Scholar.js index 7ccf8367fbb..3bab1814baf 100644 --- a/Google Scholar.js +++ b/Google Scholar.js @@ -9,13 +9,14 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-02-25 01:19:36" + "lastUpdated": "2023-07-11 07:58:52" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2022 Simon Kornblith, Frank Bennett, Aurimas Vinckevicius + Copyright © 2022 Simon Kornblith, Frank Bennett, Aurimas Vinckevicius, and + Zoë C. Ma. This file is part of Zotero. @@ -35,6 +36,31 @@ ***** END LICENSE BLOCK ***** */ +const DELAY_INTERVAL = 2000; // in milliseconds + +var GS_CONFIG = { baseURL: undefined, lang: undefined }; + +const MIME_TYPES = { + PDF: 'application/pdf', + DOC: 'application/msword', + HTML: 'text/html', +}; + +// The only "typedef" that needs to be kept in mind: a data object representing +// a row in the seach/profile listing. + +/** + * Information object for one Google Scholar entry or "row" + * + * @typedef {Object} RowObj + * @property {?string} id - Google Scholar ID string + * @property {string} [directLink] - href of the title link + * @property {string} [attachmentLink] - href of the attachment link found by GS + * @property {string} [attachmentType] - type (file extension) of the attachment + * @property {string} [byline] - the line of text below the title (in green) + */ + + /* Detection for law cases, but not "How cited" pages, * e.g. url of "how cited" page: * http://scholar.google.co.jp/scholar_case?about=1101424605047973909&q=kelo&hl=en&as_sdt=2002 @@ -74,12 +100,12 @@ function getSearchResults(doc, checkOnly) { var found = false; var rows = doc.querySelectorAll('.gs_r[data-cid]'); for (var i = 0; i < rows.length; i++) { - var href = rows[i].dataset.cid; + var id = rows[i].dataset.cid; var title = text(rows[i], '.gs_rt'); - if (!href || !title) continue; + if (!id || !title) continue; if (checkOnly) return true; found = true; - items[href] = title; + items[id] = title; } return found ? items : false; } @@ -101,198 +127,233 @@ function getProfileResults(doc, checkOnly) { } -function doWeb(doc, url) { - var type = detectWeb(doc, url); +async function doWeb(doc, url) { + // Determine the domain and language variant of the page. + let urlObj = new URL(url); + GS_CONFIG.baseURL = urlObj.origin; + GS_CONFIG.lang = urlObj.searchParams.get("hl") || "en"; + + let type = detectWeb(doc, url); + if (type == "multiple") { - if (getSearchResults(doc, true)) { - Zotero.selectItems(getSearchResults(doc, false), function (items) { - if (!items) { - return; - } - var ids = []; - for (var i in items) { - ids.push(i); - } - // here it is enough to know the ids and we can call scrape directly - scrape(doc, ids); - }); + let referrerURL; + let getRow; + let keys; + + if (getSearchResults(doc, true/* checkOnly */)) { + let items = await Z.selectItems(getSearchResults(doc, false)); + if (!items) { + return; + } + referrerURL = new URL(doc.location); + getRow = rowFromSearchResult; + keys = Object.keys(items); } - else if (getProfileResults(doc, true)) { - Zotero.selectItems(getProfileResults(doc, false), function (items) { - if (!items) { - return; - } - var articles = []; - for (var i in items) { - articles.push(i); - } - // here we need open these pages before calling scrape - ZU.processDocuments(articles, scrape); - }); + else if (getProfileResults(doc, true/* checkOnly */)) { + let urls = await Z.selectItems(getProfileResults(doc, false)); + if (!urls) { + return; + } + const profileName = text(doc, "#gsc_prf_in"); + referrerURL = getEmulatedSearchURL(profileName); + getRow = rowFromProfile; + keys = Object.keys(urls); } + + await scrapeMany(keys, doc, getRow, referrerURL); } else { // e.g. https://scholar.google.de/citations?view_op=view_citation&hl=de&user=INQwsQkAAAAJ&citation_for_view=INQwsQkAAAAJ:u5HHmVD_uO8C - scrape(doc, url, type); + await scrape(doc, url, type); } } -function scrape(doc, idsOrUrl, type) { - if (Array.isArray(idsOrUrl)) { - scrapeIds(doc, idsOrUrl); +// Scrape an array of string IDs or URLs (keys) that are obtained from +// the GS search/profile document (baseDocument). rowRequestor is a function +// that returns the row or a promise resolving to a row when called as +// rowRequestor(key, baseDocument). +// This function will reject if some rows failed to translate. +async function scrapeMany(keys, baseDocument, rowRequestor, referrerURL) { + let failedRows = []; + let promises = []; + for (let i = 0; i < keys.length; i++) { + let key = keys[i]; + let row = await rowRequestor(key, baseDocument); + if (row) { + // NOTE: here we start a promise that scrapes the row in the stages + // of DOI -> arXiv -> Google Scholar, but don't wait for it in the + // loop over rows + promises.push(scrapeInStages(row, referrerURL, failedRows)); + } + if (i < keys.length - 1) { + // But we do wait between iterations over the rows + await delay(DELAY_INTERVAL); + } } - else if (type && type == "case") { - scrapeCase(doc, idsOrUrl); + await Promise.all(promises); + if (failedRows.length) { + throw new Error(`${failedRows.length} row(s) failed to translate`); + } +} + + +// Scrape one GS entry +async function scrape(doc, url, type) { + if (type && type == "case") { + scrapeCase(doc, url); } else { - var related = ZU.xpathText(doc, '//a[contains(@href, "q=related:")]/@href'); - if (!related) { - throw new Error("Could not locate related URL"); - } - var itemID = related.match(/=related:([^:]+):/); - if (itemID) { - scrapeIds(doc, [itemID[1]]); + // Stand-alone "View article" page + const profileName = text(doc, "#gsc_sb_ui > div > a"); + let referrerURL = getEmulatedSearchURL(profileName); + // Single-item row computed from "View article" page content. + let row = parseViewArticle(doc); + if (row) { + let failedRow = []; + await scrapeInStages(row, referrerURL, failedRow); + if (failedRow.length) { + throw new Error(`Failed to translate: ${row}`); + } } else { - Z.debug("Can't find itemID. related URL is " + related); - throw new Error("Cannot extract itemID from related link"); + throw new Error(`Expected 'View article' page at ${url}, but failed to extract article info from it.`); } } } +// "row requestor" functions +// For search results - given ID and the document it originates, return a row. +// This function does not incur additional network requests. +function rowFromSearchResult(id, doc) { + try { + let entryElem = doc.querySelector(`.gs_r[data-cid="${id}"]`); + // href from an tag, direct link to the source. Note that the ID + // starting with number can be fine, but the selector is a pain. + let aElem = doc.getElementById(id); + let directLink = aElem ? aElem.href : undefined; + let attachmentLink = attr(entryElem, ".gs_ggs a", "href"); + let attachmentType = text(entryElem, ".gs_ctg2"); + if (attachmentType) { + // Remove the brackets + attachmentType = attachmentType.slice(1, -1).toUpperCase(); + } + let byline = text(entryElem, ".gs_a"); -function scrapeIds(doc, ids) { - for (let i = 0; i < ids.length; i++) { - // We need here 'let' to access ids[i] later in the nested functions - let context = doc.querySelector('.gs_r[data-cid="' + ids[i] + '"]'); - if (!context && ids.length == 1) context = doc; - var citeUrl = '/scholar?q=info:' + ids[i] + ':scholar.google.com/&output=cite&scirp=1'; - // For 'My Library' we check the search field at the top - // and then in these cases change the citeUrl accordingly. - var scilib = attr(doc, '#gs_hdr_frm input[name="scilib"]', 'value'); - if (scilib && scilib == 1) { - citeUrl = '/scholar?scila=' + ids[i] + '&output=cite&scirp=1'; + return { id, directLink, attachmentLink, attachmentType, byline }; + } + catch (error) { + Z.debug(`Warning: failed to get row info for GS id ${id}`); + return undefined; + } +} + +// For search results - given "Article view" URLs and the profile document it +// originates, return a row. This will incur one request (to get the "Article +// view" document) per row. +async function rowFromProfile(url, profileDoc) { + // To "navigate" to the linked "View article" page from the profile page, a + // referrer is sent as header in the request + const requestOptions = { headers: { Referer: profileDoc.location.href } }; + + try { + let viewArticleDoc = await requestDocument(url, requestOptions); + let row = parseViewArticle(viewArticleDoc); + if (row) { + return row; } - ZU.doGet(citeUrl, function (citePage) { - var m = citePage.match(/href="((https?:\/\/[a-z.]*)?\/scholar.bib\?[^"]+)/); - if (!m) { - // Saved lists and possibly other places have different formats for BibTeX URLs - // Trying to catch them here (can't add test bc lists are tied to google accounts) - m = citePage.match(/href="(.+?)">BibTeX<\/a>/); - } - if (!m) { - var msg = "Could not find BibTeX URL"; - var title = citePage.match(/(.*?)<\/title>/i); - if (title) { - if (title) msg += ' Got page with title "' + title[1] + '"'; - } - throw new Error(msg); - } - var bibUrl = ZU.unescapeHTML(m[1]); - ZU.doGet(bibUrl, function (bibtex) { - var translator = Zotero.loadTranslator("import"); - translator.setTranslator("9cb70025-a888-4a29-a210-93ec52da40d4"); - translator.setString(bibtex); - translator.setHandler("itemDone", function (obj, item) { - // these two variables are extracted from the context - var titleLink = attr(context, 'h3 a, #gsc_vcd_title a', 'href'); - var secondLine = text(context, '.gs_a') || ''; - // case are not recognized and can be characterized by the - // titleLink, or that the second line starts with a number - // e.g. 1 Cr. 137 - Supreme Court, 1803 - if ((titleLink && titleLink.includes('/scholar_case?')) - || secondLine && ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'].includes(secondLine[0])) { - item.itemType = "case"; - item.caseName = item.title; - item.reporter = item.publicationTitle; - item.reporterVolume = item.volume; - item.dateDecided = item.date; - item.court = item.publisher; - } - // patents are not recognized but are easily detected - // by the titleLink or second line - if ((titleLink && titleLink.includes('google.com/patents/')) || secondLine.includes('Google Patents')) { - item.itemType = "patent"; - // authors are inventors - for (let i = 0, n = item.creators.length; i < n; i++) { - item.creators[i].creatorType = 'inventor'; - } - // country and patent number - if (titleLink) { - let m = titleLink.match(/\/patents\/([A-Za-z]+)(.*)$/); - if (m) { - item.country = m[1]; - item.patentNumber = m[2]; - } - } - } - - // fix titles in all upper case, e.g. some patents in search results - if (item.title.toUpperCase() == item.title) { - item.title = ZU.capitalizeTitle(item.title); - } - - // delete "others" as author - if (item.creators.length) { - var lastCreatorIndex = item.creators.length - 1, - lastCreator = item.creators[lastCreatorIndex]; - if (lastCreator.lastName === "others" && (lastCreator.fieldMode === 1 || lastCreator.firstName === "")) { - item.creators.splice(lastCreatorIndex, 1); - } - } - - // clean author names - for (let j = 0, m = item.creators.length; j < m; j++) { - if (!item.creators[j].firstName) continue; - - item.creators[j] = ZU.cleanAuthor( - item.creators[j].lastName + ', ' - + item.creators[j].firstName, - item.creators[j].creatorType, - true); - } - - // attach linked document as attachment if available - var documentLinkTarget = attr(context, '.gs_or_ggsm a, #gsc_vcd_title_gg a', 'href'); - var documentLinkTitle = text(context, '.gs_or_ggsm a, #gsc_vcd_title_gg a'); - if (documentLinkTarget) { - // Z.debug(documentLinkTarget); - var attachment = { - title: "Full Text", - url: documentLinkTarget - }; - let m = documentLinkTitle.match(/^\[(\w+)\]/); - if (m) { - var mimeTypes = { - PDF: 'application/pdf', - DOC: 'application/msword', - HTML: 'text/html' - }; - if (Object.keys(mimeTypes).includes(m[1].toUpperCase())) { - attachment.mimeType = mimeTypes[m[1]]; - } - } - item.attachments.push(attachment); - } - - // Attach linked page as snapshot if available - if (titleLink && titleLink != documentLinkTarget) { - item.attachments.push({ - url: titleLink, - title: "Snapshot", - mimeType: "text/html" - }); - } + } + catch (error) { + Z.debug(`Warning: cannot retrieve the profile view-article page at ${url}; skipping. The error was:`); + Z.debug(error); + return undefined; + } - item.complete(); - }); - translator.translate(); - }); - }); + Z.debug(`Warning: cannot find Google Scholar id in profile view-article page at ${url}; skipping.`); + return undefined; +} + +// process the row in the order of DOI -> arXiv -> GS. If all fail, add the row +// to the array failedRows. This function never rejects. +async function scrapeInStages(row, referrerURL, failedRows) { + try { + await scrapeDOI(row); + return; + } + catch (error) { + } + + try { + await scrapeArXiv(row); + return; + } + catch (error) { + } + + try { + await scrapeGoogleScholar(row, referrerURL); + } + catch (error) { + Z.debug(`Error with Google Scholar scraping of row ${row.directLink}`); + Z.debug(`The error was: ${error}`); + failedRows.push(row); } } +function scrapeDOI(row) { + let doi = extractDOI(row); + if (!doi) { + throw new Error(`No DOI found for link: ${row.directLink}`); + } + + let translate = Z.loadTranslator("search"); + // DOI Content Negotiation + translate.setTranslator("b28d0d42-8549-4c6d-83fc-8382874a5cb9"); + translate.setHandler("error", () => {}); + translate.setHandler("itemDone", (obj, item) => { + // NOTE: The 'DOI Content Negotiation' translator does not add + // attachments on its own + addAttachment(item, row); + item.complete(); + }); + translate.setSearch({ DOI: doi }); + Z.debug(`Trying DOI search for ${row.directLink}`); + return translate.translate(); +} + +function scrapeArXiv(row) { + let eprintID = extractArXiv(row); + if (!eprintID) { + throw new Error(`No ArXiv eprint ID found for link: ${row.directLink}`); + } + + let translate = Z.loadTranslator("search"); + // arXiv.org + translate.setTranslator("ecddda2e-4fc6-4aea-9f17-ef3b56d7377a"); + translate.setHandler("error", () => {}); + translate.setHandler("itemDone", (obj, item) => { + // NOTE: Attachment is handled by the arXiv.org search translator + item.complete(); + }); + translate.setSearch({ arXiv: eprintID }); + Z.debug(`Trying ArXiv search for ${row.directLink}`); + return translate.translate(); +} + +function scrapeGoogleScholar(row, referrerURL) { + // URL of the citation-info page fragment for the current row + let citeURL; + + if (referrerURL.searchParams.get("scilib") === "1") { // My Library + citeURL = `${GS_CONFIG.baseURL}/scholar?scila=${row.id}&output=cite&scirp=0&hl=${GS_CONFIG.lang}`; + } + else { // Normal search page + citeURL = `${GS_CONFIG.baseURL}/scholar?q=info:${row.id}:scholar.google.com/&output=cite&scirp=0&hl=${GS_CONFIG.lang}`; + } + + Z.debug(`Falling back to Google Scholar scraping for ${row.directLink || "citation-only entry"}`); + return processCitePage(citeURL, row, referrerURL.href); +} /* * ######################### @@ -661,6 +722,237 @@ ItemFactory.prototype.saveItemCommonVars = function () { } }; + +/* + * ######################### + * ### Utility Functions ### + * ######################### + */ + +// Returns a promise that resolves (to undefined) after the minimum time delay +// specified in milliseconds +function delay(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +// Identification functions for external searches + +/** + * Extract candidate DOI from row by parsing its direct-link URL + * + * @param {RowObj} row + * @returns {string?} Candidate DOI string, or null if not found + */ +function extractDOI(row) { + let path = decodeURIComponent((new URL(row.directLink)).pathname); + // Normally, match to the end of the path, because we couldn't have known + // better. + // But we can try clean up a bit, for common file extensions tacked to the + // end, e.g. the link in the header title of + // https://scholar.google.com/citations?view_op=view_citation&hl=en&user=Cz6X6UYAAAAJ&citation_for_view=Cz6X6UYAAAAJ:zYLM7Y9cAGgC + // https://www.nomos-elibrary.de/10.5771/9783845229614-153.pdf + let m = path.match(/(10\.\d{4,}\/.+?)(?:[./](?:pdf|htm|html|xhtml|epub|xml))?$/i); + return m && m[1]; +} + +/** + * Extract arXiv ID from row by parsing its direct-link URL + * + * @param {RowObj} row + * @returns {string?} ArXiv ID, or null if not found + */ +function extractArXiv(row) { + let urlObj = new URL(row.directLink); + if (urlObj.hostname.toLowerCase() !== "arxiv.org") { + return null; + } + let path = decodeURIComponent(urlObj.pathname); + let m = path.match(/\/\w+\/([a-z-]+\/\d+|\d+\.\d+)$/i); + return m && m[1]; +} + +// Page-processing utilities + +/** + * Returns an emulated search URL for a GS search with the profile name as the + * search term + * + * @param {string} profileName - Name of the profile's owner + * @returns {URL} + */ +function getEmulatedSearchURL(profileName) { + return new URL(`/scholar?hl=${GS_CONFIG.lang}&as_sdt=0%2C5&q=${encodeURIComponent(profileName).replace(/%20/g, "+")}&btnG=`, GS_CONFIG.baseURL); +} + +/** + * Parse the "View article" page and returns the equivalent of a GS + * search-result row + * + * @param {Document} viewArticleDoc - "View article" document + * @returns {RowObj?} The row object, or null if parsing failed. + */ +function parseViewArticle(viewArticleDoc) { + let related = ZU.xpathText(viewArticleDoc, + '//a[contains(@href, "q=related:")]/@href'); + if (!related) { + Z.debug("Could not locate 'related' link on the 'View article' page."); + return null; + } + + let m = related.match(/=related:([^:]+):/); // GS id + if (m) { + let id = m[1]; + let directLink = attr(viewArticleDoc, ".gsc_oci_title_link", "href"); + let attachmentLink = attr(viewArticleDoc, "#gsc_oci_title_gg a", "href"); + let attachmentType = text(viewArticleDoc, ".gsc_vcd_title_ggt"); + if (attachmentType) { + attachmentType = attachmentType.slice(1, -1).toUpperCase(); + } + return { id, directLink, attachmentLink, attachmentType }; + } + else { + Z.debug("Unexpected format of 'related' URL; can't find Google Scholar id. 'related' URL is " + related); + return null; + } +} + +/** + * Request and read the page-fragment with citation info, retrieve BibTeX, and + * import. Each call sends two network requests, and each request is preceded + * by a delay. + * + * @param {string} citeURL - The citation-info page fragment's URL, to be + * requested. + * @param {RowObj} row - The row object carrying the information of the entry's + * identity. + * @param {string} referrer - The referrer for the citation-info page fragment + * request. + */ +async function processCitePage(citeURL, row, referrer) { + let requestOptions = { headers: { Referer: referrer } }; + // Note that the page at citeURL has no doctype and is not a complete HTML + // document. The browser can parse it in quirks mode but ZU.requestDocument + // has trouble with it. + await delay(DELAY_INTERVAL); + const citePage = await requestText(citeURL, requestOptions); + + let m = citePage.match(/href="((https?:\/\/[a-z.]*)?\/scholar.bib\?[^"]+)/); + if (!m) { + // Saved lists and possibly other places have different formats for + // BibTeX URLs + // Trying to catch them here (can't add test bc lists are tied to + // google accounts) + m = citePage.match(/href="(.+?)">BibTeX<\/a>/); + } + if (!m) { + var msg = "Could not find BibTeX URL"; + var title = citePage.match(/<title>(.*?)<\/title>/i); + if (title) { + msg += ' Got page with title "' + title[1] + '"'; + } + throw new Error(msg); + } + const bibTeXURL = ZU.unescapeHTML(m[1]); + + // Pause between obtaining the citation info page and sending the request + // for the BibTeX document + await delay(DELAY_INTERVAL); + + // NOTE: To emulate the web app, the referrer for the BibTeX text is always + // set to the origin (e.g. https://scholar.google.com/), imitating + // strict-origin-when-cross-origin + requestOptions.headers.Referer = GS_CONFIG.baseURL + "/"; + const bibTeXBody = await requestText(bibTeXURL, requestOptions); + + let translator = Z.loadTranslator("import"); + translator.setTranslator("9cb70025-a888-4a29-a210-93ec52da40d4"); // BibTeX + translator.setString(bibTeXBody); + translator.setHandler("itemDone", function (obj, item) { + // case are not recognized and can be characterized by the + // title link, or that the second line starts with a number + // e.g. 1 Cr. 137 - Supreme Court, 1803 + if ((row.directLink && row.directLink.includes('/scholar_case?')) + || row.byline && "01234567890".includes(row.byline[0])) { + item.itemType = "case"; + item.caseName = item.title; + item.reporter = item.publicationTitle; + item.reporterVolume = item.volume; + item.dateDecided = item.date; + item.court = item.publisher; + } + // patents are not recognized but are easily detected + // by the titleLink or second line + if ((row.directLink && row.directLink.includes('google.com/patents/')) + || (row.byline && row.byline.includes('Google Patents'))) { + item.itemType = "patent"; + // authors are inventors + for (let i = 0, n = item.creators.length; i < n; i++) { + item.creators[i].creatorType = 'inventor'; + } + // country and patent number + if (row.directLink) { + let m = row.directLink.match(/\/patents\/([A-Za-z]+)(.*)$/); + if (m) { + item.country = m[1]; + item.patentNumber = m[2]; + } + } + } + + // Add the title link as the url of the item + if (row.directLink) { + item.url = row.directLink; + } + + // fix titles in all upper case, e.g. some patents in search results + if (item.title.toUpperCase() === item.title) { + item.title = ZU.capitalizeTitle(item.title); + } + + // delete "others" as author + if (item.creators.length) { + var lastCreatorIndex = item.creators.length - 1, + lastCreator = item.creators[lastCreatorIndex]; + if (lastCreator.lastName === "others" && (lastCreator.fieldMode === 1 || lastCreator.firstName === "")) { + item.creators.splice(lastCreatorIndex, 1); + } + } + + // clean author names + for (let j = 0, m = item.creators.length; j < m; j++) { + if (!item.creators[j].firstName) { + continue; + } + + item.creators[j] = ZU.cleanAuthor( + item.creators[j].lastName + ', ' + + item.creators[j].firstName, + item.creators[j].creatorType, + true); + } + + addAttachment(item, row); + + item.complete(); + }); + return translator.translate(); +} + +function addAttachment(item, row) { + // attach linked document as attachment if available + if (row.attachmentLink) { + let attachment = { + title: "Available Version (via Google Scholar)", + url: row.attachmentLink, + }; + let mimeType = MIME_TYPES[row.attachmentType]; + if (mimeType) { + attachment.mimeType = mimeType; + } + item.attachments.push(attachment); + } +} + /* Test Case Descriptions: (these have not been included in the test case JSON below as per aurimasv's comment on https://github.com/zotero/translators/pull/833) @@ -942,10 +1234,11 @@ var testCases = [ { "type": "web", "url": "https://scholar.google.de/citations?view_op=view_citation&hl=de&user=INQwsQkAAAAJ&citation_for_view=INQwsQkAAAAJ:u5HHmVD_uO8C", + "detectedItemType": "journalArticle", "items": [ { - "itemType": "journalArticle", - "title": "Linked data-the story so far", + "itemType": "bookSection", + "title": "Linked data: The story so far", "creators": [ { "firstName": "Christian", @@ -963,17 +1256,17 @@ var testCases = [ "creatorType": "author" } ], - "date": "2009", - "itemID": "bizer2009linked", + "date": "2011", + "bookTitle": "Semantic services, interoperability and web applications: emerging concepts", + "itemID": "bizer2011linked", "libraryCatalog": "Google Scholar", "pages": "205–227", - "publicationTitle": "Semantic services, interoperability and web applications: emerging concepts", + "publisher": "IGI global", + "shortTitle": "Linked data", + "url": "https://www.igi-global.com/chapter/linkeddata-story-far/55046", "attachments": [ { - "title": "Snapshot" - }, - { - "title": "Fulltext", + "title": "Available Version (via Google Scholar)", "mimeType": "application/pdf" } ], @@ -1023,6 +1316,46 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://scholar.google.com/citations?view_op=view_citation&hl=en&user=RjsFKYEAAAAJ&cstart=20&pagesize=80&citation_for_view=RjsFKYEAAAAJ:5nxA0vEk-isC", + "detectedItemType": "journalArticle", + "items": [ + { + "itemType": "journalArticle", + "title": "The Weakness of Power and the Power of Weakness: The Ethics of War in a Time of Terror", + "creators": [ + { + "creatorType": "author", + "firstName": "Michael", + "lastName": "Northcott" + } + ], + "date": "04/2007", + "DOI": "10.1177/0953946806075493", + "ISSN": "0953-9468, 1745-5235", + "abstractNote": "In 2002 a significant number of American theologians declared that the ‘war on terror’ was a just war. But the indiscriminate strategies and munitions technologies deployed in the invasion and occupation of Iraq fall short of the just war principles of non-combatant immunity, and proportionate response. The just war tradition is one of Christendom's most enduring legacies to the law of nations. Its practice implies a standard of virtue in war that is undermined by the indiscriminate effects of many modern weapons and by the deliberate targeting of civilian infrastructure. The violent power represented by the technology of what the Vatican calls ‘total war’has occasioned a significant shift in Catholic social teaching on just war since the Second World War. Total war generates an asymmetry of weakness in those subjected to these techniques of terror, and this has only strengthened the violence of the Islamist struggle against the West. But those who draw inspiration and legitimacy from this weakness in their struggle with the West also reject virtue in war. In a time of terror the theological vocation is to speak peace and to recall the terms in which the peace of God was achieved by way of the cross.", + "issue": "1", + "journalAbbreviation": "Studies in Christian Ethics", + "language": "en", + "libraryCatalog": "DOI.org (Crossref)", + "pages": "88-101", + "publicationTitle": "Studies in Christian Ethics", + "shortTitle": "The Weakness of Power and the Power of Weakness", + "url": "http://journals.sagepub.com/doi/10.1177/0953946806075493", + "volume": "20", + "attachments": [ + { + "title": "Available Version (via Google Scholar)", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/HAL Archives Ouvertes.js b/HAL Archives Ouvertes.js index dc3473add4e..fa12d66ea87 100644 --- a/HAL Archives Ouvertes.js +++ b/HAL Archives Ouvertes.js @@ -2,20 +2,20 @@ "translatorID": "f20f91fe-d875-47e7-9656-0abb928be472", "label": "HAL Archives Ouvertes", "creator": "Sebastian Karcher", - "target": "^https?://hal\\.archives-ouvertes\\.fr", + "target": "^https://(hal\\.archives-ouvertes\\.fr|hal\\.science)\\b", "minVersion": "3.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-12-12 19:29:40" + "lastUpdated": "2023-07-12 08:47:33" } /* ***** BEGIN LICENSE BLOCK ***** HAL translator - Copyright © 2012-2014 Sebastian Karcher + Copyright © 2012-2014 Sebastian Karcher and contributors This file is part of Zotero. @@ -36,51 +36,59 @@ */ function detectWeb(doc, url) { - if (url.search(/\/search\/index\//)!=-1) return "multiple"; - if (url.search(/\index\.php\?halsid=|\.fr\/[a-z]+-\d+/)!=-1) return findItemType(doc, url); + if (/\/search\/index\//.test(url)) return "multiple"; + if (/\/hal-\d+/.test(url)) return findItemType(doc, url); + return false; } -function findItemType(doc, url){ - var itemType = text(doc, 'div.label'); - //Z.debug(itemType) +function findItemType(doc, url) { + var itemType = text(doc, '.typdoc') + // do some preliminary cleaning + .split("(")[0].trim() // discard parenthesized text + .split(", ")[0].trim() // simplify "Pré-publication, Document de travail" and " Preprints, Working Papers, ..." + .toLowerCase(); var typeMap = { - "Books": "book", - "Ouvrage (y compris édition critique et traduction)": "book", - "Book sections": "bookSection", - "Chapitre d'ouvrage": "bookSection", - "Conference papers": "conferencePaper", - "Communication dans un congrès": "conferencePaper", - "Directions of work or proceedings": "book", - "Direction d'ouvrage, Proceedings": "book", - "Journal articles": "journalArticle", - "Article dans des revues": "journalArticle", - "Lectures": "presentation", - "Cours": "presentation", - "Other publications": "book", //this could also be report, not sure here but bibtex guesses book - "Autre publication": "book", //this could also be report, not sure here but bibtex guesses book - "Patents": "patent", - "Brevet": "patent", - "Preprints, Working Papers, ...": "manuscript", - "Pré-publication, Document de travail": "manuscript", - "Reports": "report", - "Rapport": "report", - "Theses": "thesis", - "Thèse": "thesis", - "Poster communications": "presentation", - "Poster de conférence": "presentation" - } + /* eslint-disable quote-props */ + "books": "book", + "ouvrages": "book", + "book sections": "bookSection", + "chapitre d'ouvrage": "bookSection", + "conference papers": "conferencePaper", + "communication dans un congrès": "conferencePaper", + "directions of work or proceedings": "book", + "direction d'ouvrage": "book", + "journal articles": "journalArticle", + "article dans une revue": "journalArticle", + "lectures": "presentation", + "cours": "presentation", + "other publications": "book", // this could also be report, not sure here but bibtex guesses book + "autre publication scientifique": "book", // this could also be report, not sure here but bibtex guesses book + "patents": "patent", + "brevet": "patent", + "preprints": "preprint", + "pré-publication": "preprint", + "reports": "report", + "rapport": "report", + "scientific blog post": "blogPost", + "article de blog scientifique": "blogPost", + "theses": "thesis", + "thèse": "thesis", + "poster communications": "presentation", + "poster de conférence": "presentation", + /* eslint-enable quote-props */ + }; if (typeMap[itemType]) return typeMap[itemType]; - else if (url.indexOf("medihal-")!=-1) return "artwork"; + else if (url.includes("medihal-")) return "artwork"; else return "journalArticle"; } function doWeb(doc, url) { - var articles = new Array(); + var articles = []; if (detectWeb(doc, url) == "multiple") { var items = {}; var titles = doc.evaluate('//strong/a[@data-original-title="Display the resource" or @data-original-title="Voir la ressource"]', doc, null, XPathResult.ANY_TYPE, null); var title; - while (title = titles.iterateNext()) { + while ((title = titles.iterateNext())/* assignment */) { items[title.href] = title.textContent; } Zotero.selectItems(items, function (items) { @@ -90,34 +98,33 @@ function doWeb(doc, url) { for (var i in items) { articles.push(i); } - Zotero.Utilities.processDocuments(articles, scrape) + Zotero.Utilities.processDocuments(articles, scrape); + return true; }); - } else { - //work on PDF pages - if (url.search(/\/document$/) != -1 ) { - var articleURL = url.replace(/\/document$/, "") - //Z.debug(articleURL) - ZU.processDocuments(articleURL, scrape); - } - else scrape(doc, url); } + else if (/\/document$/.test(url)) { // work on PDF pages + var articleURL = url.replace(/\/document$/, ""); + // Z.debug(articleURL) + ZU.processDocuments(articleURL, scrape); + } + else scrape(doc, url); } function scrape(doc, url) { var bibtexUrl = url.replace(/#.+|\/$/, "") + "/bibtex"; - var abstract = ZU.xpathText(doc, '//div[@class="abstract-content"]'); - var pdfUrl = ZU.xpathText(doc, '//meta[@name="citation_pdf_url"]/@content'); - //Z.debug("pdfURL " + pdfUrl) + var abstract = text(doc, '.abstract-content'); + var pdfUrl = attr(doc, "#viewer-detailed a[download]", "href"); + // Z.debug("pdfURL " + pdfUrl) ZU.doGet(bibtexUrl, function (bibtex) { - //Z.debug(bibtex) + // Z.debug(bibtex) var translator = Zotero.loadTranslator("import"); translator.setTranslator("9cb70025-a888-4a29-a210-93ec52da40d4"); translator.setString(bibtex); translator.setHandler("itemDone", function (obj, item) { - if (abstract){ - item.abstractNote=abstract.replace(/(Abstract|Résumé)\s*:/, ""); + if (abstract) { + item.abstractNote = abstract.replace(/^(Abstract|Résumé)\s*:/, ""); } - if (pdfUrl){ + if (pdfUrl) { item.attachments = [{ url: pdfUrl, title: "HAL PDF Full Text", @@ -141,8 +148,9 @@ function scrape(doc, url) { item.complete(); }); translator.translate(); - }) + }); } + /** BEGIN TEST CASES **/ var testCases = [ { diff --git a/Haaretz.js b/Haaretz.js index 466a8a0da21..774ea57a877 100644 --- a/Haaretz.js +++ b/Haaretz.js @@ -9,11 +9,11 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2016-10-31 17:33:48" + "lastUpdated": "2023-11-05 08:15:35" } /** - Copyright (c) 2015 Eran Rosenthal + Copyright (c) 2015 Eran Rosenthal and contributors This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License @@ -30,50 +30,198 @@ <http://www.gnu.org/licenses/>. */ + function detectWeb(doc, url) { - if (ZU.xpathText(doc, '//header//h1')) { - return 'newspaperArticle'; + let path = new URL(url).pathname; + if (/article-podcast\//.test(path)) { + return "podcast"; + } + if (/\/opinions?\/letters\//.test(path)) { + return "letter"; + } + if (/-cartoon\/|\/opinions\/caricatures\//.test(path)) { + return "artwork"; + } + // Selectors for multiple results will also match on the home page but not + // all of them point to single items. Special-case the home page to prevent + // this. This could have been dealt with better if the class names weren't + // obfuscated. + if (path === "/") return false; + let ld = getLD(doc); + if (ld && ["NewsArticle", "LiveBlogPosting"].includes(ld["@type"])) { + return "newspaperArticle"; } + return getSearchResults(doc, true) && "multiple"; } -function doWeb(doc, url) { - var item = new Zotero.Item('newspaperArticle'); - item.title = ZU.xpathText(doc, '//header//h1'); - item.url = url; - if (url.indexOf('haaretz.com') != -1) { - item.publicationTitle = 'Haaretz'; - item.language = 'en'; - } else { - item.publicationTitle = 'הארץ'; - item.language = 'he'; +function getSearchResults(doc, checkOnly) { + let url = doc.location.href; + if (/^https:\/\/[^/]+\/search-results($|\?)/.test(url)) { + if (checkOnly) { // only observe in detection stage; otherwise an error + let root = doc.getElementById("__next"); + if (root) Z.monitorDOMChanges(root); + } + return getSiteSearchContent(doc, checkOnly); } + else { + return getSectionContent(doc, checkOnly); + } +} - var abstract = ZU.xpathText(doc, '//header/p'); - if (!abstract) abstract = ZU.xpathText(doc, '//meta[@property="og:description"]/@content'); - item.abstractNote = abstract; +function getSiteSearchContent(doc, checkOnly) { + let items = {}; + let found = false; + let rows = doc.querySelectorAll('article header a'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} - var authors = ZU.xpath(doc, '//address/a[@rel="author"]'); - for (var i=0; i<authors.length; i++) { - item.creators.push(ZU.cleanAuthor(authors[i].textContent, 'author')); +function getSectionContent(doc, checkOnly) { + let items = {}; + let found = false; + let rows = doc.querySelectorAll('main section a[href^="/"]'); + for (let row of rows) { + if (!row.querySelector("h1, h2, h3") && !(row.parentElement && row.parentElement.tagName === "LI")) continue; + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; } + return found ? items : false; +} - item.date = ZU.strToISO(ZU.xpathText(doc, '//time[@itemprop="datePublished"]/@datetime')); - var keywords = ZU.xpathText(doc, '//meta[@name="news_keywords"]/@content').split(','); - for (var i=0; i<keywords.length; i++) { - if (keywords[i].length>0) item.tags.push(keywords[i].trim()); +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); } - item.complete(); +} + +async function scrape(doc, url = doc.location.href) { + let translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + + translator.setHandler('itemDone', (_obj, item) => { + item.libraryCatalog = "Haaretz"; + + let ld = getLD(doc); + + // replace creators; EM fails for multiple authors + if (ld.author) { + item.creators = ld.author + .filter(obj => obj["@type"] === "Person") + .map(obj => ZU.cleanAuthor(obj.name, "author")); + } + + // find section by breadcrumb but only if breadcrumb does not stop at + // top level + if (ld.breadcrumb && ld.breadcrumb.itemListElement.length > 1) { + let breadCrumb = ld.breadcrumb.itemListElement.slice(-1)[0].name; + if (breadCrumb) { + item.section = breadCrumb; + } + } + else { + item.section = ""; + } + + // title (headline); EM sometimes gives headline that contains more + // noise + if (ld.headline) { + item.title = ld.headline.replace(/ [-|] .+$/, ""); + } + + // abstract (abstract content in the <meta> elements are inconveniently + // capitalized) + if (ld.description) { + item.abstractNote = ld.description; + } + else { + let lede = text(doc, "main header h1 + p"); + if (lede) { + item.abstractNote = lede; + } + } + + // Fix language field for Arabic content + let sample = item.title || item.abstractNote || ""; + if (/[\u0600-\u06ff]/.test(sample)) { + item.language = "ar"; + } + // Note that the Arabic-language content also falls under the Hebrew + // publicationTitle for some reason + let langIsEn = /^en/i.test(item.language || "en"); + item.publicationTitle = langIsEn ? "Haaretz" : "הארץ"; + + // Fix authorship for editorial articles + if (/\/opinions?\/editorial/.test(item.url || doc.location.href)) { + item.creators = []; + } + + // Fix authorship, container title, and length for podcasts + if (item.itemType === "podcast") { + let firstCreator = item.creators[0]; + if (firstCreator) { + let podcastTitle = (firstCreator.firstName || "") + (firstCreator.lastName ? ` ${firstCreator.lastName}` : ""); + if (podcastTitle) { + item.seriesTitle = podcastTitle; + } + } + item.creators = []; + let runningTime = attr(doc, 'main header div[role="slider"]', "aria-valuemax"); + if (runningTime) { + item.runningTime = runningTime; + } + } + + if (item.itemType === "letter") { + item.letterType = langIsEn ? "Letter to the editor" : "מכתב לעורך"; + delete item.section; + // there's no easy way of detecting the author reliably for + // English-language letters + if (langIsEn) item.creators = []; + } + + item.complete(); + }); + + let em = await translator.getTranslatorObject(); + em.itemType = detectWeb(doc, url) || "newspaperArticle"; + await em.doWeb(doc, url); +} + +function getLD(doc) { + let ldScript = text(doc, "script[type='application/ld+json']"); + if (ldScript) return JSON.parse(ldScript); + return null; } /** BEGIN TEST CASES **/ var testCases = [ { "type": "web", - "url": "http://www.haaretz.com/israel-news/1.671202", + "url": "https://www.haaretz.com/2015-08-14/ty-article/islamic-jihad-if-hunger-striker-dies-well-respond-with-force/0000017f-f0b6-d223-a97f-fdff11760000", "items": [ { "itemType": "newspaperArticle", - "title": "Islamic Jihad: If Hunger Striker Dies, We'll Respond With Force Against Israel", + "title": "Islamic Jihad: If hunger striker dies, we'll respond with force against Israel", "creators": [ { "firstName": "Jack", @@ -97,10 +245,17 @@ var testCases = [ "libraryCatalog": "Haaretz", "publicationTitle": "Haaretz", "shortTitle": "Islamic Jihad", - "url": "http://www.haaretz.com/israel-news/1.671202", - "attachments": [], + "url": "https://www.haaretz.com/2015-08-14/ty-article/islamic-jihad-if-hunger-striker-dies-well-respond-with-force/0000017f-f0b6-d223-a97f-fdff11760000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], "tags": [ - "Palestinian hunger strike" + { + "tag": "Palestinian hunger strike" + } ], "notes": [], "seeAlso": [] @@ -109,7 +264,7 @@ var testCases = [ }, { "type": "web", - "url": "http://www.haaretz.co.il/news/politics/1.2708080", + "url": "https://www.haaretz.co.il/news/politics/2015-08-15/ty-article/0000017f-e675-da9b-a1ff-ee7f93440000", "items": [ { "itemType": "newspaperArticle", @@ -131,18 +286,277 @@ var testCases = [ "language": "he", "libraryCatalog": "Haaretz", "publicationTitle": "הארץ", - "url": "http://www.haaretz.co.il/news/politics/1.2708080", - "attachments": [], + "section": "מדיני ביטחוני", + "url": "https://www.haaretz.co.il/news/politics/2015-08-15/ty-article/0000017f-e675-da9b-a1ff-ee7f93440000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], "tags": [ - "טרור", - "פיגוע", - "פלסטינים", - "צה\"ל" + { + "tag": "טרור" + }, + { + "tag": "פיגוע" + }, + { + "tag": "פלסטינים" + }, + { + "tag": "צה\"ל" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.haaretz.com/search-results?q=cuisine", + "defer": true, + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.haaretz.com/science-and-health/climate-change", + "defer": true, + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.haaretz.com/ty-tag/lgbt-0000017f-da2a-d42c-afff-dffad1ae0000", + "defer": true, + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.haaretz.co.il/debate/2023-10-26/ty-article/0000018b-6b3b-de3d-abdb-7f7b1ba00000", + "items": [ + { + "itemType": "newspaperArticle", + "title": "أمام مشاهد الدمار، في غزة بدأوا يشككون بالقرار الذي اتخذته حماس", + "creators": [ + { + "firstName": "جاكي", + "lastName": "خوري", + "creatorType": "author" + } + ], + "date": "2023-10-26", + "abstractNote": "سكان القطاع بدأوا يقولون بصوت مرتفع إن حماس أخطأت في تقدير المخاطر المترتبة عن الهجوم على إسرائيل ـ وربما تكون ارتكبت خطأ مصيرياً أول مَن يعاني مِن جرّائه هم السكان الذين يتلقّون، بأجسادهم انتقام الجيش الإسرائيلي", + "language": "ar", + "libraryCatalog": "Haaretz", + "publicationTitle": "הארץ", + "section": "הארץ בערבית", + "url": "https://www.haaretz.co.il/debate/2023-10-26/ty-article/0000018b-6b3b-de3d-abdb-7f7b1ba00000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.haaretz.co.il/opinions/editorial-articles/2023-11-01/ty-article-opinion/0000018b-8658-d055-afbf-b6fb84690000", + "items": [ + { + "itemType": "newspaperArticle", + "title": "ארדן, פרובוקציה עלובה", + "creators": [], + "date": "2023-11-01", + "language": "he", + "libraryCatalog": "Haaretz", + "publicationTitle": "הארץ", + "section": "מאמר מערכת", + "url": "https://www.haaretz.co.il/opinions/editorial-articles/2023-11-01/ty-article-opinion/0000018b-8658-d055-afbf-b6fb84690000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "גלעד ארדן" + }, + { + "tag": "האומות המאוחדות - האו\"ם" + }, + { + "tag": "מלחמת חרבות ברזל" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.haaretz.co.il/digital/podcast/2023-10-22/ty-article-podcast/0000018b-5742-db77-afdb-dfc2efea0000", + "items": [ + { + "itemType": "podcast", + "title": "28 דקות של אסקפיזם שמתחילות בשירה האס ונגמרות בסמבוסק חשאי", + "creators": [], + "abstractNote": "תרבות יום א', פודקאסט התרבות של \"הארץ\" עם גילי איזיקוביץ וניב הדס: 28 דקות של אסקפיזם שהתחילו בסדרה החדשה של שירה האס ובמחמאות לא צפויות לנטפליקס.", + "language": "he", + "runningTime": "28:00", + "seriesTitle": "תרבות יום א'", + "url": "https://www.haaretz.co.il/digital/podcast/2023-10-22/ty-article-podcast/0000018b-5742-db77-afdb-dfc2efea0000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.haaretz.com/israel-news/podcasts/2022-10-06/ty-article-podcast/.premium/will-religious-voters-decide-who-is-israels-next-leader-listen-to-election-overdose/00000183-ae7e-d0e7-a7a3-fefee5440000", + "items": [ + { + "itemType": "podcast", + "title": "Will religious voters decide who Israel's next leader is? LISTEN to Election Overdose", + "creators": [], + "abstractNote": "Israel's national religious sector could play a key role in Israel's November 1 election, and parties all across the political spectrum are seeking the religious vote. Haaretz's Election Overdose podcast tries to make sense of the religious voter's dilemma", + "language": "en", + "seriesTitle": "Election Overdose", + "shortTitle": "Will religious voters decide who Israel's next leader is?", + "url": "https://www.haaretz.com/israel-news/podcasts/2022-10-06/ty-article-podcast/.premium/will-religious-voters-decide-who-is-israels-next-leader-listen-to-election-overdose/00000183-ae7e-d0e7-a7a3-fefee5440000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Israeli elections" + }, + { + "tag": "Israeli politics" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.haaretz.com/opinion/letters/2019-09-03/ty-article-opinion/letters-to-the-editor-were-hiding-a-girl-to-protect-her-from-deportation/0000017f-ecf5-d0f7-a9ff-eef59efa0000", + "items": [ + { + "itemType": "letter", + "title": "We are hiding a Filipino girl to protect her from deportation", + "creators": [], + "date": "2019-09-03", + "abstractNote": "The child we are hiding My partner and I live in the Tel Aviv area. We are sheltering a foreign worker from the Philippines and her 12-year-old daughter. All our friends", + "language": "en", + "letterType": "Letter to the editor", + "libraryCatalog": "Haaretz", + "url": "https://www.haaretz.com/opinion/letters/2019-09-03/ty-article-opinion/letters-to-the-editor-were-hiding-a-girl-to-protect-her-from-deportation/0000017f-ecf5-d0f7-a9ff-eef59efa0000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "LGBTQ" + }, + { + "tag": "Migrant workers" + }, + { + "tag": "Russia" + }, + { + "tag": "Vladimir Putin" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.haaretz.co.il/opinions/letters/2023-10-31/ty-article-opinion/.premium/0000018b-85eb-d805-a98f-b5fb8d4e0000", + "items": [ + { + "itemType": "letter", + "title": "הופקרנו", + "creators": [ + { + "firstName": "נועה", + "lastName": "אצילי", + "creatorType": "author" + } + ], + "date": "2023-10-31", + "language": "he", + "letterType": "מכתב לעורך", + "libraryCatalog": "Haaretz", + "url": "https://www.haaretz.co.il/opinions/letters/2023-10-31/ty-article-opinion/.premium/0000018b-85eb-d805-a98f-b5fb8d4e0000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.haaretz.co.il/opinions/caricatures/2023-10-18/ty-article-opinion/.premium/0000018b-3d97-dd29-a3df-fdf735970000", + "items": [ + { + "itemType": "artwork", + "title": "קריקטורה יומית", + "creators": [ + { + "firstName": "ערן", + "lastName": "וולקובסקי", + "creatorType": "author" + } + ], + "date": "2023-10-18", + "abstractNote": "הארץ", + "language": "he", + "libraryCatalog": "Haaretz", + "url": "https://www.haaretz.co.il/opinions/caricatures/2023-10-18/ty-article-opinion/.premium/0000018b-3d97-dd29-a3df-fdf735970000", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } ], + "tags": [], "notes": [], "seeAlso": [] } ] } ] -/** END TEST CASES **/ \ No newline at end of file +/** END TEST CASES **/ diff --git a/IEEE Xplore.js b/IEEE Xplore.js index 92e350fc2ad..d258e8ea445 100644 --- a/IEEE Xplore.js +++ b/IEEE Xplore.js @@ -9,10 +9,9 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-03-02 14:47:46" + "lastUpdated": "2023-09-24 03:17:24" } - /* ***** BEGIN LICENSE BLOCK ***** @@ -37,6 +36,7 @@ */ function detectWeb(doc, url) { + Zotero.monitorDOMChanges(doc.querySelector('.global-content-wrapper')); if (doc.defaultView !== null && doc.defaultView !== doc.defaultView.top) return false; if (/[?&]arnumber=(\d+)/i.test(url) || /\/document\/\d+/i.test(url)) { @@ -111,28 +111,25 @@ function fixUrl(url) { } } -function doWeb(doc, url) { - if (detectWeb(doc, url) == "multiple") { - Zotero.selectItems(getSearchResults(doc), function (items) { - if (!items) { - return; - } - var articles = []; - for (var i in items) { - articles.push(i); - } - ZU.processDocuments(articles, scrape); - }); + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } } else if (url.includes("/search/") || url.includes("/stamp/") || url.includes("/ielx4/") || url.includes("/ielx5/")) { - ZU.processDocuments([fixUrl(url)], scrape); + await scrape(await requestDocument([fixUrl(url)])); } else { - scrape(doc, url); + await scrape(doc, url); } } -function scrape(doc, url) { + +async function scrape(doc, url = doc.location.href) { var arnumber = (url.match(/arnumber=(\d+)/) || url.match(/\/document\/(\d+)/))[1]; var pdf = "/stamp/stamp.jsp?tp=&arnumber=" + arnumber; // Z.debug("arNumber = " + arnumber); @@ -151,98 +148,101 @@ function scrape(doc, url) { } - var post = "recordIds=" + arnumber + "&fromPage=&citations-format=citation-abstract&download-format=download-bibtex"; - ZU.doPost('/xpl/downloadCitations', post, function (text) { - text = ZU.unescapeHTML(text.replace(/(&[^\s;]+) and/g, '$1;')); - // remove empty tag - we can take this out once empty tags are ignored - text = text.replace(/(keywords=\{.+);\}/, "$1}"); - var earlyaccess = false; - if (text.search(/^@null/) != -1) { - earlyaccess = true; - text = text.replace(/^@null/, "@article"); + let bibtexURL = "/rest/search/citation/format?recordIds=" + arnumber + "&fromPage=&citations-format=citation-abstract&download-format=download-bibtex"; + Z.debug(bibtexURL); + // metadata is downloaded in a JSON data field + let bibtex = await requestJSON(bibtexURL, { headers: { Referer: url} }); + bibtex = bibtex.data; + bibtex = ZU.unescapeHTML(bibtex.replace(/(&[^\s;]+) and/g, '$1;')); + // remove empty tag - we can take this out once empty tags are ignored + bibtex = bibtex.replace(/(keywords=\{.+);\}/, "$1}"); + var earlyaccess = false; + if (/^@null/.test(bibtex)) { + earlyaccess = true; + bibtex = text.replace(/^@null/, "@article"); + } + var translator = Zotero.loadTranslator("import"); + // Calling the BibTeX translator + translator.setTranslator("9cb70025-a888-4a29-a210-93ec52da40d4"); + translator.setString(bibtex); + translator.setHandler("itemDone", function (obj, item) { + item.notes = []; + var res; + // Rearrange titles, per http://forums.zotero.org/discussion/8056 + // If something has a comma or a period, and the text after comma ends with + // "of", "IEEE", or the like, then we switch the parts. Prefer periods. + if (item.publicationTitle.includes(".")) { + res = item.publicationTitle.trim().match(/^(.*)\.(.*(?:of|on|IEE|IEEE|IET|IRE))$/); + } + else { + res = item.publicationTitle.trim().match(/^(.*),(.*(?:of|on|IEE|IEEE|IET|IRE))$/); + } + if (res) { + item.publicationTitle = res[2] + " " + res[1]; + } + item.proceedingsTitle = item.conferenceName = item.publicationTitle; + if (earlyaccess) { + item.volume = "Early Access Online"; + item.issue = ""; + item.pages = ""; } - var translator = Zotero.loadTranslator("import"); - // Calling the BibTeX translator - translator.setTranslator("9cb70025-a888-4a29-a210-93ec52da40d4"); - translator.setString(text); - translator.setHandler("itemDone", function (obj, item) { - item.notes = []; - var res; - // Rearrange titles, per http://forums.zotero.org/discussion/8056 - // If something has a comma or a period, and the text after comma ends with - // "of", "IEEE", or the like, then we switch the parts. Prefer periods. - if (item.publicationTitle.includes(".")) { - res = item.publicationTitle.trim().match(/^(.*)\.(.*(?:of|on|IEE|IEEE|IET|IRE))$/); - } - else { - res = item.publicationTitle.trim().match(/^(.*),(.*(?:of|on|IEE|IEEE|IET|IRE))$/); - } - if (res) { - item.publicationTitle = res[2] + " " + res[1]; - } - item.proceedingsTitle = item.conferenceName = item.publicationTitle; - if (earlyaccess) { - item.volume = "Early Access Online"; - item.issue = ""; - item.pages = ""; - } - - if (data && data.authors && data.authors.length == item.creators.length) { - item.creators = []; - for (let author of data.authors) { - item.creators.push({ - firstName: author.firstName, - lastName: author.lastName, - creatorType: "author" - }); - } - } - if (!item.ISSN && data && data.issn) { - item.ISSN = data.issn.map(el => el.value).join(", "); - } - if (item.ISSN && !ZU.fieldIsValidForType('ISSN', item.itemType)) { - item.extra = "ISSN: " + item.ISSN; + if (data && data.authors && data.authors.length == item.creators.length) { + item.creators = []; + for (let author of data.authors) { + item.creators.push({ + firstName: author.firstName, + lastName: author.lastName, + creatorType: "author" + }); } + } - item.attachments.push({ - document: doc, - title: "IEEE Xplore Abstract Record" - }); + if (!item.ISSN && data && data.issn) { + item.ISSN = data.issn.map(el => el.value).join(", "); + } + if (item.ISSN && !ZU.fieldIsValidForType('ISSN', item.itemType)) { + item.extra = "ISSN: " + item.ISSN; + } + item.url = url; + item.attachments.push({ + document: doc, + title: "IEEE Xplore Abstract Record" + }); - if (pdf) { - ZU.doGet(pdf, function (src) { - // Either the PDF is embedded in the page, or (e.g. for iOS) - // the page has a redirect to the full-page PDF - // - // As of 3/2020, embedded PDFs via a web-based proxy are - // being served as getPDF.jsp, so support that in addition - // to direct .pdf URLs. - var m = /<i?frame src="([^"]+\.pdf\b[^"]*|[^"]+\/getPDF\.jsp\b[^"]*)"|<meta HTTP-EQUIV="REFRESH" content="0; url=([^\s"]+\.pdf\b[^\s"]*)"/.exec(src); - var pdfUrl = m && (m[1] || m[2]); - if (pdfUrl) { - item.attachments.unshift({ - url: pdfUrl, - title: "IEEE Xplore Full Text PDF", - mimeType: "application/pdf" - }); - } - item.complete(); - }, null); - } - else { + if (pdf) { + ZU.doGet(pdf, function (src) { + // Either the PDF is embedded in the page, or (e.g. for iOS) + // the page has a redirect to the full-page PDF + // + // As of 3/2020, embedded PDFs via a web-based proxy are + // being served as getPDF.jsp, so support that in addition + // to direct .pdf URLs. + var m = /<i?frame src="([^"]+\.pdf\b[^"]*|[^"]+\/getPDF\.jsp\b[^"]*)"|<meta HTTP-EQUIV="REFRESH" content="0; url=([^\s"]+\.pdf\b[^\s"]*)"/.exec(src); + var pdfUrl = m && (m[1] || m[2]); + if (pdfUrl) { + item.attachments.unshift({ + url: pdfUrl, + title: "IEEE Xplore Full Text PDF", + mimeType: "application/pdf" + }); + } item.complete(); - } - }); + }, null); + } + else { + item.complete(); + } + }); - translator.getTranslatorObject(function (trans) { - trans.setKeywordSplitOnSpace(false); - trans.setKeywordDelimRe('\\s*;\\s*', ''); - trans.doImport(); - }); + translator.getTranslatorObject(function (trans) { + trans.setKeywordSplitOnSpace(false); + trans.setKeywordDelimRe('\\s*;\\s*', ''); + trans.doImport(); }); } + /** BEGIN TEST CASES **/ var testCases = [ { diff --git a/IPCC.js b/IPCC.js index 7cd2e432b7c..ed02fd27a84 100644 --- a/IPCC.js +++ b/IPCC.js @@ -9,13 +9,13 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-08-03 22:06:14" + "lastUpdated": "2023-09-18 01:10:08" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2021 Abe Jellinek + Copyright © 2021 Abe Jellinek and contributors This file is part of Zotero. @@ -141,6 +141,75 @@ let report6Wg2Ch17Authors = 'Mark New; Diana Reckien; David Viner; Carolina Adle let report6Wg2Ch18Authors = 'E. Lisa F. Schipper; Aromar Revi; Benjamin L. Preston; Edward. R. Carr; Siri H. Eriksen; Luis R. Fernández-Carril; Bruce Glavovic; Nathalie J.M. Hilmi; Debora Ley; Rupa Mukerji; M. Silvia Muylaert de Araujo; Rosa Perez; Steven K. Rose; Pramod K. Singh' .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); +let report6Wg3Editors = "Priyadarshi R. Shukla; Jim Skea; Raphael Slade; Al Khourdajie, Alaa; van Diemen, Renée; David McCollum; Minal Pathak; Shreya Some; Purvi Vyas; Roger Fradera; Malek Belkacemi; Apoorva Hasija; Géninha Lisboa; Sigourney Luz; Juliette Malley" + .split('; ').map(name => ZU.cleanAuthor(name, 'editor', name.includes(', '))); + +let report6Wg3SFPAuthors = "Jim Skea; Priyadarshi R. Shukla; Andy Reisinger; Raphael Slade; Minal Pathak; Al Khourdajie, Alaa; van Diemen, Renée; Amjad Abdulla; Keigo Akimoto; Mustafa Babiker; Quan Bai; Igor A. Bashmakov; Christopher Bataille; Göran Berndes; Gabriel Blanco; Kornelis Blok; Mercedes Bustamante; Edward Byers; Luisa F. Cabeza; Katherine Calvin; Carlo Carraro; Leon Clarke; Annette Cowie; Felix Creutzig; Diriba Korecha Dadi; Dipak Dasgupta; de Coninck, Heleen; Fatima Denton; Shobhakar Dhakal; Navroz K. Dubash; Oliver Geden; Michael Grubb; Céline Guivarch; Shreekant Gupta; Andrea N. Hahmann; Kirsten Halsnaes; Paulina Jaramillo; Kejun Jiang; Frank Jotzo; Tae Yong Jung; Kahn Ribeiro, Suzana; Smail Khennas; Şiir Kılkış; Silvia Kreibiehl; Volker Krey; Elmar Kriegler; William F. Lamb; Franck Lecocq; Shuaib Lwasa; Nagmeldin Mahmoud; Cheikh Mbow; David McCollum; Jan Christoph Minx; Catherine Mitchell; Rachid Mrabet; Yacob Mulugetta; Gert-Jan Nabuurs; Gregory F. Nemet; Peter Newman; Leila Niamir; Lars J. Nilsson; Sudarmanto Budi Nugroho; Chukwumerije Okereke; Shonali Pachauri; Anthony Patt; Ramón Pichs-Madruga; Joana Portugal-Pereira; Lavanya Rajamani; Keywan Riahi; Joyashree Roy; Yamina Saheb; Roberto Schaeffer; Karen C. Seto; Shreya Some; Linda Steg; Ferenc L. Toth; Diana Ürge-Vorsatz; van Vuuren, Detlef P.; Elena Verdolini; Purvi Vyas; Yi-Ming Wei; Mariama Williams; Harald Winkler" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3TechSummaryAuthors = "Minal Pathak; Raphael Slade; Ramón Pichs-Madruga; Diana Ürge-Vorsatz; Priyadarshi R. Shukla; Jim Skea; Amjad Abdulla; Al Khourdajie, Alaa; Mustafa Babiker; Quan Bai; Igor A. Bashmakov; Christopher Bataille; Göran Berndes; Gabriel Blanco; Luisa F. Cabeza; Carlo Carraro; Leon Clarke; de Coninck, Heleen; Felix Creutzig; Diriba Korecha Dadi; Fatima Denton; Shobhakar Dhakal; van Diemen, Renée; Navroz K. Dubash; Amit Garg; Oliver Geden; Michael Grubb; Céline Guivarch; Kirsten Halsnaes; Paulina Jaramillo; Tae Yong Jung; Kahn Ribeiro, Suzana; Şiir Kılkış; Alexandre Koberle; Silvia Kreibiehl; Elmar Kriegler; William F. Lamb; Franck Lecocq; Shuaib Lwasa; Nagmeldin Mahmoud; Eric Masanet; David McCollum; Jan Christoph Minx; Catherine Mitchell; Kanako Morita; Rachid Mrabet; Gert-Jan Nabuurs; Peter Newman; Leila Niamir; Lars J. Nilsson; Chukwumerije Okereke; Anthony Patt; Joana Portugal-Pereira; Lavanya Rajamani; Andy Reisinger; Keywan Riahi; Joyashree Roy; Ambuj Sagar; Yamina Saheb; Roberto Schaeffer; Karen C. Seto; Pete Smith; Shreya Some; Benjamin K. Sovacool; Linda Steg; Massimo Tavoni; Ferenc L. Toth; Purvi Vyas; Yi-Ming Wei; Jake Whitehead; Thomas Wiedmann; Harald Winkler" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch1Authors = "Michael Grubb; Chukwumerije Okereke; Jun Arima; Valentina Bosetti; Ying Chen; James Edmonds; Shreekant Gupta; Alexandre Köberle; Snorre Kverndokk; Arunima Malik; Linda Yanti Sulistiawati" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch2Authors = "Shobhakar Dhakal; Jan Christoph Minx; Ferenc L. Toth; Amr Abdel-Aziz; Figueroa Meza, Maria Josefina; Klaus Hubacek; Inge G.C. Jonckheere; Yong-Gun Kim; Gregory F. Nemet; Shonali Pachauri; Xianchun C. Tan; Thomas Wiedmann" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch3Authors = "Keywan Riahi; Roberto Schaeffer; Jacobo Arango; Katherine Calvin; Céline Guivarch; Tomoko Hasegawa,; Kejun Jiang; Elmar Kriegler; Robert Matthews; Glen P. Peters; Anand Rao; Simon Robertson; Adam Mohammed Sebbit; Julia Steinberger; Massimo Tavoni; van Vuuren, Detlef P." + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch4Authors = "Franck Lecocq; Harald Winkler; Julius Partson Daka; Sha Fu; James S. Gerber; Sivan Kartha; Volker Krey; Hans Lofgren; Toshihiko Masui; Ritu Mathur; Joana Portugal-Pereira; Benjamin K. Sovacool; Maria Virginia Vilariño; Nan Zhou" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch5Authors = "Felix Creutzig; Joyashree Roy; Patrick Devine-Wright; Julio Díaz-José; Frank W. Geels; Arnulf Grubler; Nadia Maïzi; Eric Masanet; Yacob Mulugetta; Chioma Daisy Onyige; Patricia E. Perkins; Alessandro Sanches-Pereira; Elke Ursula Weber" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch6Authors = "Leon Clarke; Yi-Ming Wei; De La Vega Navarro, Angel; Amit Garg; Andrea N. Hahmann; Smail Khennas; Lima de Azevedo, Inês Margarida; Andreas Löschel; Ajay Kumar Singh; Linda Steg; Goran Strbac; Kenichi Wada" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch7Authors = "Gert-Jan Nabuurs; Rachid Mrabet; Abu Hatab, Assem; Mercedes Bustamante; Harry Clark; Petr Havlík; Joanna I. House; Cheikh Mbow; Karachepone N. Ninan; Alexander Popp; Stephanie Roe; Brent Sohngen; Sirintornthep Towprayoon" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch8Authors = "Shuaib Lwasa; Karen C. Seto; Xuemei Bai; Hilda Blanco; Kevin R. Gurney; Şiir Kılkış; Oswaldo Lucon; Jin Murakami; Jiahua Pan; Ayyoob Sharifi; Yoshiki Yamagata" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch9Authors = "Luisa F. Cabeza; Quan Bai; Paolo Bertoldi; Jacob M. Kihila; André F.P. Lucena; Érika Mata; Sebastian Mirasgedis; Aleksandra Novikova; Yamina Saheb" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch10Authors = "Paulina Jaramillo; Kahn Ribeiro, Suzana; Peter Newman; Subash Dhar; Ogheneruona E. Diemuodeke; Tsutomu Kajino; David Simon Lee; Sudarmanto Budi Nugroho; Xunmin Ou; Anders Hammer Strømman; Jake Whitehead" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch11Authors = "Igor A. Bashmakov; Lars J. Nilsson; Adolf Acquaye; Christopher Bataille; Jonathan M. Cullen; de la Rue du Can, Stéphane; Manfred Fischedick; Yong Geng; Kanako Tanaka" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch12Authors = "Mustafa Babiker; Göran Berndes; Kornelis Blok; Brett Cohen; Annette Cowie; Oliver Geden; Veronika Ginzburg; Adrian Leip; Pete Smith; Masahiro Sugiyama; Francis Yamba" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch13Authors = "Navroz K. Dubash; Catherine Mitchell; Elin Lerum Boasson; Mercy J. Borbor-Córdova; Solomone Fifita; Erik Haites; Mark Jaccard; Frank Jotzo; Sasha Naidoo; Patricia Romero-Lankao; Wei Shen; Mykola Shlapak; Libo Wu" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch14Authors = "Anthony Patt; Lavanya Rajamani; Preety Bhandari; Antonina Ivanova Boncheva; Alejandro Caparrós; Kamal Djemouai; Izumi Kubota; Jacqueline Peel; Agus Pratama Sari; Detlef F. Sprinz; Jørgen Wettestad" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch15Authors = "Silvia Kreibiehl; Tae Yong Jung; Stefano Battiston; Carvajal Sarzosa, Pablo Esteban; Christa Clapp; Dipak Dasgupta; Nokuthula Dube; Raphaël Jachnik; Kanako Morita; Nahla Samargandi; Mariama Williams" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch16Authors = "Gabriel Blanco; de Coninck, Heleen; Lawrence Agbemabiese; El Hadji Mbaye Diagne; Diaz Anadon, Laura; Yun Seng Lim; Walter Alberto Pengue; Ambuj D. Sagar; Taishi Sugiyama; Kenji Tanaka; Elena Verdolini; Jan Witajewski-Baltvilks" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Ch17Authors = "Fatima Denton; Kirsten Halsnæs; Keigo Akimoto; Sarah Burch; Diaz Morejon, Cristobal; Fernando Farias; Joni Jupesta; Ali Shareef; Petra Schweizer-Ries; Fei Teng; Eric Zusman" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Annex1Authors = "van Diemen, Renée; J.B. Robin Matthews; Vincent Möller; Jan S. Fuglestvedt; Valérie Masson-Delmotte; Carlos Méndez; Andy Reisinger; Sergey Semenov" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Annex2Authors = "Al Khourdajie, Alaa; van Diemen, Renée; William F. Lamb; Minal Pathak; Andy Reisinger; de la Rue du Can, Stéphane; Jim Skea; Raphael Slade; Shreya Some; Linda Steg" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + +let report6Wg3Annex3Authors = "Céline Guivarch; Elmar Kriegler; Joana Portugal-Pereira, Valentina Bosetti; James Edmonds; Manfred Fischedick; Petr Havlík; Paulina Jaramillo; Volker Krey; Franck Lecocq; André F.P. Lucena; Malte Meinshausen; Sebastian Mirasgedis; Brian O’Neill; Glen P. Peters; Joeri Rogelj; Steven Rose; Yamina Saheb; Goran Strbac; Anders Hammer Strømman; van Vuuren, Detlef P.; Nan Zhou" + .split('; ').map(name => ZU.cleanAuthor(name, 'author', name.includes(', '))); + function wg1Chapter(slug, itemTemplate) { return Object.assign(new Zotero.Item('bookSection'), itemTemplate, { bookTitle: 'Climate Change 2021: The Physical Science Basis. Contribution of Working Group I to the Sixth Assessment Report of the Intergovernmental Panel on Climate Change', @@ -175,6 +244,23 @@ function wg2Chapter(slug, itemTemplate) { }); } +function wg3Chapter(slug, itemTemplate) { + return Object.assign(new Zotero.Item('bookSection'), itemTemplate, { + bookTitle: 'Climate Change 2022: Mitigation of Climate Change. Contribution of Working Group III to the Sixth Assessment Report of the Intergovernmental Panel on Climate Change', + publisher: 'Cambridge University Press', + place: "Cambridge, UK and New York, NY, USA", + date: '2022', + creators: [...(itemTemplate.creators || []), ...report6Wg3Editors], + attachments: [ + { + title: 'Full Text PDF', + mimeType: 'application/pdf', + url: `https://www.ipcc.ch/report/ar6/wg3/downloads/report/IPCC_AR6_WGIII_${slug}.pdf` + } + ] + }); +} + let citations = { '/report/ar6/wg1/': { 'Full Report': Object.assign(new Zotero.Item('book'), { @@ -359,7 +445,7 @@ let citations = { { title: 'Full Text PDF', mimeType: 'application/pdf', - url: 'https://www.ipcc.ch/report/ar6/wg2/downloads/report/IPCC_AR6_WGII_FinalDraft_FullReport.pdf' + url: 'https://report.ipcc.ch/ar6/wg2/IPCC_AR6_WGII_FullReport.pdf' } ] }), @@ -372,79 +458,244 @@ let citations = { abstractNote: 'The Technical Summary (TS) provides extended summary of key findings and serves as a link between the comprehensive assessment of the Working Group II Report and the concise SPM.', creators: report6Wg2TechSummaryAuthors }), - 'Chapter 1: Point of departure and key concepts': wg2Chapter('FinalDraft_Chapter01', { + 'Chapter 1: Point of departure and key concepts': wg2Chapter('Chapter01', { title: 'Point of departure and key concepts', creators: report6Wg2Ch1Authors }), - 'Chapter 2: Terrestrial and freshwater ecosystems and their services': wg2Chapter('FinalDraft_Chapter02', { + 'Chapter 2: Terrestrial and freshwater ecosystems and their services': wg2Chapter('Chapter02', { title: 'Terrestrial and freshwater ecosystems and their services', creators: report6Wg2Ch2Authors }), - 'Chapter 3: Ocean and coastal ecosystems and their services': wg2Chapter('FinalDraft_Chapter03', { + 'Chapter 3: Ocean and coastal ecosystems and their services': wg2Chapter('Chapter03', { title: 'Ocean and coastal ecosystems and their services', creators: report6Wg2Ch3Authors }), - 'Chapter 4: Water': wg2Chapter('FinalDraft_Chapter04', { + 'Chapter 4: Water': wg2Chapter('Chapter04', { title: 'Water', creators: report6Wg2Ch4Authors }), - 'Chapter 5: Food, fibre, and other ecosystem products': wg2Chapter('FinalDraft_Chapter05', { + 'Chapter 5: Food, fibre, and other ecosystem products': wg2Chapter('Chapter05', { title: 'Food, fibre, and other ecosystem products', creators: report6Wg2Ch5Authors }), - 'Chapter 6: Cities, settlements and key infrastructure': wg2Chapter('FinalDraft_Chapter06', { + 'Chapter 6: Cities, settlements and key infrastructure': wg2Chapter('Chapter06', { title: 'Cities, settlements and key infrastructure', creators: report6Wg2Ch6Authors }), - 'Chapter 7: Health, wellbeing and the changing structure of communities': wg2Chapter('FinalDraft_Chapter07', { + 'Chapter 7: Health, wellbeing and the changing structure of communities': wg2Chapter('Chapter07', { title: 'Health, wellbeing and the changing structure of communities', creators: report6Wg2Ch7Authors }), - 'Chapter 8: Poverty, livelihoods and sustainable development': wg2Chapter('FinalDraft_Chapter08', { + 'Chapter 8: Poverty, livelihoods and sustainable development': wg2Chapter('Chapter08', { title: 'Poverty, livelihoods and sustainable development', creators: report6Wg2Ch8Authors }), - 'Chapter 9: Africa': wg2Chapter('FinalDraft_Chapter09', { + 'Chapter 9: Africa': wg2Chapter('Chapter09', { title: 'Africa', creators: report6Wg2Ch9Authors }), - 'Chapter 10: Asia': wg2Chapter('FinalDraft_Chapter10', { + 'Chapter 10: Asia': wg2Chapter('Chapter10', { title: 'Asia', creators: report6Wg2Ch10Authors }), - 'Chapter 11: Australasia': wg2Chapter('FinalDraft_Chapter11', { + 'Chapter 11: Australasia': wg2Chapter('Chapter11', { title: 'Australasia', creators: report6Wg2Ch11Authors }), - 'Chapter 12: Central and South America': wg2Chapter('FinalDraft_Chapter12', { + 'Chapter 12: Central and South America': wg2Chapter('Chapter12', { title: 'Central and South America', creators: report6Wg2Ch12Authors }), - 'Chapter 13: Europe': wg2Chapter('FinalDraft_Chapter13', { + 'Chapter 13: Europe': wg2Chapter('Chapter13', { title: 'Europe', creators: report6Wg2Ch13Authors }), - 'Chapter 14: North America': wg2Chapter('FinalDraft_Chapter14', { + 'Chapter 14: North America': wg2Chapter('Chapter14', { title: 'North America', creators: report6Wg2Ch14Authors }), - 'Chapter 15: Small Islands': wg2Chapter('FinalDraft_Chapter15', { + 'Chapter 15: Small Islands': wg2Chapter('Chapter15', { title: 'Small Islands', creators: report6Wg2Ch15Authors }), - 'Chapter 16: Key risks across sectors and regions': wg2Chapter('FinalDraft_Chapter16', { + 'Chapter 16: Key risks across sectors and regions': wg2Chapter('Chapter16', { title: 'Key risks across sectors and regions', creators: report6Wg2Ch16Authors }), - 'Chapter 17: Decision-making options for managing risk': wg2Chapter('FinalDraft_Chapter17', { + 'Chapter 17: Decision-making options for managing risk': wg2Chapter('Chapter17', { title: 'Decision-making options for managing risk', creators: report6Wg2Ch17Authors }), - 'Chapter 18: Climate resilient development pathways': wg2Chapter('FinalDraft_Chapter18', { + 'Chapter 18: Climate resilient development pathways': wg2Chapter('Chapter18', { title: 'Climate resilient development pathways', creators: report6Wg2Ch18Authors }) - } + }, + "/report/ar6/wg3/": { + 'Full Report': Object.assign(new Zotero.Item('book'), { + title: 'Climate Change 2022: Impacts, Adaptation and Vulnerability. Contribution of Working Group III to the Sixth Assessment Report of the Intergovernmental Panel on Climate Change.', + date: '2022', + creators: report6Wg3Editors, + ISBN: "978-92-9169-160-9", + DOI: "10.1017/9781009157926", + attachments: [ + { + title: 'Full Text PDF', + mimeType: 'application/pdf', + url: 'https://www.ipcc.ch/report/ar6/wg3/downloads/report/IPCC_AR6_WGIII_FullReport.pdf' + } + ] + }), + 'Summary for Policymakers': wg3Chapter('SummaryForPolicymakers', { + title: 'Summary for Policymakers', + abstractNote: 'The Summary for Policymakers (SPM) provides a high-level summary of the key findings of the Working Group III Report and is approved by the IPCC member governments line by line.', + DOI: "10.1017/9781009157926.001", + pages: "1-48", + creators: report6Wg3SFPAuthors + }), + 'Technical Summary': wg3Chapter('TS', { + title: 'Technical Summary', + abstractNote: 'The Technical Summary (TS) provides extended summary of key findings and serves as a link between the comprehensive assessment of the Working Group III Report and the concise SPM..', + DOI: "10.1017/9781009157926.002", + pages: "49-147", + creators: report6Wg3TechSummaryAuthors + }), + 'Chapter 1: Introduction and Framing': wg3Chapter('Chapter01', { + title: 'Introduction and Framing', + DOI: "10.1017/9781009157926.003", + pages: "151-213", + creators: report6Wg3Ch1Authors + }), + 'Chapter 2: Emissions trends and drivers': wg3Chapter('Chapter02', { + title: 'Emissions Trends and Drivers', + DOI: "10.1017/9781009157926.004", + pages: "215-294", + creators: report6Wg3Ch2Authors + }), + 'Chapter 3: Mitigation pathways compatible with long-term goals': wg3Chapter('Chapter03', { + title: 'Mitigation pathways compatible with long-term goals', + DOI: "10.1017/9781009157926.005", + pages: "295-408", + creators: report6Wg3Ch3Authors + }), + 'Chapter 4: Mitigation and development pathways in the near- to mid-term': wg3Chapter('Chapter04', { + title: 'Mitigation and development pathways in the near- to mid-term', + DOI: "10.1017/9781009157926.006", + pages: "409-502", + creators: report6Wg3Ch4Authors + }), + 'Chapter 5: Demand, services and social aspects of mitigation': wg3Chapter('Chapter05', { + title: 'Demand, services and social aspects of mitigation', + DOI: "10.1017/9781009157926.007", + pages: "503-612", + creators: report6Wg3Ch5Authors + }), + 'Chapter 6: Energy systems': wg3Chapter('Chapter06', { + title: 'Energy Systems', + DOI: "10.1017/9781009157926.008", + pages: "613-746", + creators: report6Wg3Ch6Authors + }), + 'Chapter 7: Agriculture, Forestry, and Other Land Uses (AFOLU)': wg3Chapter('Chapter07', { + title: 'Agriculture, Forestry and Other Land Uses (AFOLU)', + DOI: "10.1017/9781009157926.009", + pages: "747-860", + creators: report6Wg3Ch7Authors + }), + 'Chapter 8: Urban systems and other settlements': wg3Chapter('Chapter08', { + title: 'Urban systems and other settlements', + DOI: "10.1017/9781009157926.010", + pages: "861-952", + creators: report6Wg3Ch8Authors + }), + 'Chapter 9: Buildings': wg3Chapter('Chapter09', { + title: 'Buildings', + DOI: "10.1017/9781009157926.011", + pages: "953-1048", + creators: report6Wg3Ch9Authors + }), + 'Chapter 10: Transport': wg3Chapter('Chapter10', { + title: 'Transport', + DOI: "10.1017/9781009157926.012", + pages: "1049-1160", + creators: report6Wg3Ch10Authors + }), + 'Chapter 11: Industry': wg3Chapter('Chapter11', { + title: 'Industry', + DOI: "10.1017/9781009157926.013", + pages: "1161-1243", + creators: report6Wg3Ch11Authors + }), + 'Chapter 12: Cross sectoral perspectives': wg3Chapter('Chapter12', { + title: 'Cross-sectoral perspectives', + DOI: "10.1017/9781009157926.014", + pages: "1245-1354", + creators: report6Wg3Ch12Authors + }), + 'Chapter 13: National and sub-national policies and institution': wg3Chapter('Chapter13', { + title: 'National and sub-national policies and institution', + DOI: "10.1017/9781009157926.015", + pages: "1355-1450", + creators: report6Wg3Ch13Authors + }), + 'Chapter 14: International cooperation': wg3Chapter('Chapter14', { + title: 'International cooperation', + DOI: "10.1017/9781009157926.016", + pages: "1451-1545", + creators: report6Wg3Ch14Authors + }), + 'Chapter 15: Investment and finance': wg3Chapter('Chapter15', { + title: 'Investment and finance', + DOI: "10.1017/9781009157926.017", + pages: "1547-1640", + creators: report6Wg3Ch15Authors + }), + 'Chapter 16: Innovation, technology development and transfer': wg3Chapter('Chapter16', { + title: 'Innovation, technology development and transfer', + DOI: "10.1017/9781009157926.018", + pages: "1641-1725", + creators: report6Wg3Ch16Authors + }), + 'Chapter 17: Accelerating the transition in the context of sustainable development': wg3Chapter('Chapter17', { + title: 'Accelerating the transition in the context of sustainable development', + DOI: "10.1017/9781009157926.019", + pages: "1727-1790", + creators: report6Wg3Ch17Authors + }), + 'Annex I: Glossary': wg3Chapter('Annex-I', { + title: 'Annex I: Glossary', + DOI: "10.1017/9781009157926.020", + pages: "1793-1820", + creators: report6Wg3Annex1Authors, + }), + 'Annex II: Definitions, units and conventions': wg3Chapter('Annex-II', { + title: 'Annex II: Definitions, Units and Conventions', + DOI: "10.1017/9781009157926.021", + pages: "1821-1840", + creators: report6Wg3Annex2Authors, + }), + 'Annex III: Scenarios and modelling methods': wg3Chapter('Annex-III', { + title: 'Annex III: Scenarios and modelling methods', + DOI: "10.1017/9781009157926.022", + pages: "1841-1908", + creators: report6Wg3Annex3Authors, + }), + 'Annex IV: Contributors to the IPCC Working Group III Sixth Assessment Report': wg3Chapter('Annex-IV', { + DOI: "10.1017/9781009157926.023", + pages: "1909-1924", + title: 'Annex IV: Contributors to the IPCC Working Group III Sixth Assessment Report' + }), + 'Annex V: Expert Reviewers of the IPCC WGIII Sixth Assessment Report': wg3Chapter('Annex-V', { + DOI: "10.1017/9781009157926.024", + pages: "1925-1968", + title: 'Annex V: Expert Reviewers of the IPCC WGIII Sixth Assessment Report' + }), + 'Annex VI: Acronyms': wg3Chapter('Annex-VI', { + DOI: "10.1017/9781009157926.024", + pages: "1969-1977", + title: 'Annex VI: Acronyms' + }), + }, }; function getCitations(doc) { @@ -455,6 +706,9 @@ function getCitations(doc) { else if (pathname == '/report/sixth-assessment-report-working-group-ii/') { pathname = '/report/ar6/wg2/'; } + else if (pathname == '/report/sixth-assessment-report-working-group-3/') { + pathname = '/report/ar6/wg3/'; + } return citations[pathname]; } @@ -493,6 +747,11 @@ var testCases = [ "type": "web", "url": "https://www.ipcc.ch/report/ar6/wg2/", "items": "multiple" + }, + { + "type": "web", + "url": "https://www.ipcc.ch/report/ar6/wg3/", + "items": "multiple" } ] /** END TEST CASES **/ diff --git a/Library Catalog (RERO ILS).js b/Library Catalog (RERO ILS).js index 7fef843a5ba..3ac9d7086d7 100644 --- a/Library Catalog (RERO ILS).js +++ b/Library Catalog (RERO ILS).js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-05-09 18:12:21" + "lastUpdated": "2023-08-17 15:05:49" } /* @@ -75,7 +75,15 @@ async function detectWeb(doc, url) { } if (getDomainAndID(url).length) { - return 'book'; + let type = attr('#thumbnail > img[src*="icon_"]', 'src').match(/icon_([^.]+)/); + if (type) { + return TYPE_MAPPING[type[1]] + // Assume that "online" items are articles and everything else is some kind of book + || (type[1] === 'docmaintype_online' ? 'journalArticle' : 'book'); + } + else { + return 'book'; + } } else if (getSearchResults(doc, true)) { return 'multiple'; @@ -147,11 +155,32 @@ async function scrape([domain, id]) { if (json.contribution && json.contribution.length) { for (let creator of json.contribution) { + let metadata; + if (creator.agent) { + if (creator.agent.preferred_name) { + metadata = creator.agent; + } + else { + metadata = (await requestJSON(creator.agent.$ref)).metadata; + } + } + else if (creator.entity) { + if (creator.entity.preferred_name) { + metadata = creator.entity; + } + else { + metadata = (await requestJSON(creator.entity.$ref)).metadata; + } + } + else { + Zotero.debug('Invalid creator'); + Zotero.debug(creator); + continue; + } let type = CREATOR_MAPPING[creator.role[0]] || 'author'; if (type === 'SKIP') continue; - let name = creator.agent.preferred_name - || (await requestJSON(creator.agent.$ref)).metadata.preferred_name; - if (creator.agent.type == 'bf:Organisation') { + let name = metadata.preferred_name; + if (metadata.type == 'bf:Organisation') { item.creators.push({ lastName: name, creatorType: type, @@ -352,12 +381,7 @@ var testCases = [ "place": "Paris", "publisher": "Armand Colin", "shortTitle": "Histoire économique de la France", - "attachments": [ - { - "title": "Catalog Entry", - "mimeType": "text/html" - } - ], + "attachments": [], "tags": [], "notes": [ { @@ -390,14 +414,9 @@ var testCases = [ "language": "eng", "libraryCatalog": "Library Catalog (RERO ILS)", "numPages": "XV, 251", - "place": "New York (N.Y.)", + "place": "New York", "publisher": "Van Nostrand Reinhold", - "attachments": [ - { - "title": "Catalog Entry", - "mimeType": "text/html" - } - ], + "attachments": [], "tags": [ { "tag": "Bender-Gestalt Test" @@ -453,10 +472,6 @@ var testCases = [ "title": "Full Text PDF", "mimeType": "application/pdf", "snapshot": false - }, - { - "title": "Catalog Entry", - "mimeType": "text/html" } ], "tags": [], @@ -504,25 +519,22 @@ var testCases = [ "creatorType": "contributor" }, { - "lastName": "The Beecham choral society ( London )", + "lastName": "The Beecham choral society (London, 1955-1962)", "creatorType": "contributor", "fieldMode": 1 }, { - "lastName": "Royal Philharmonic Orchestra ( Londres )", + "lastName": "Royal Philharmonic Orchestra (Londres)", "creatorType": "contributor", "fieldMode": 1 } ], + "callNumber": "R006481802", "label": "Columbia, P", "language": "eng", "libraryCatalog": "Library Catalog (RERO ILS)", - "attachments": [ - { - "title": "Catalog Entry", - "mimeType": "text/html" - } - ], + "place": "Lieu de publication non identifié", + "attachments": [], "tags": [], "notes": [], "seeAlso": [] @@ -560,18 +572,13 @@ var testCases = [ "numPages": "X, 166", "place": "Buckinghamshire", "publisher": "Open university", - "attachments": [ - { - "title": "Catalog Entry", - "mimeType": "text/html" - } - ], + "attachments": [], "tags": [ { - "tag": "Mixed economy - Great Britain" + "tag": "Mixed economy" }, { - "tag": "Social service - Government policy - Great Britain" + "tag": "Social service" } ], "notes": [], @@ -601,18 +608,13 @@ var testCases = [ "numPages": "xxv, 413", "place": "Thousand Oaks", "publisher": "Sage", - "attachments": [ - { - "title": "Catalog Entry", - "mimeType": "text/html" - } - ], + "attachments": [], "tags": [ { - "tag": "Cities and towns - Cross-cultural studies" + "tag": "Cities and towns" }, { - "tag": "Metropolitan areas - Cross-cultural studies" + "tag": "Metropolitan areas" }, { "tag": "Sociology, Urban" @@ -651,6 +653,7 @@ var testCases = [ } ], "date": "2014", + "callNumber": "R008116813", "label": "Tôt ou Tard", "language": "eng", "libraryCatalog": "Library Catalog (RERO ILS)", @@ -665,6 +668,35 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://bib.rero.ch/rbnj/documents/627927", + "items": [ + { + "itemType": "book", + "title": "Bakŝis, skizoj el la vivo de Egiptoj : novelaro", + "creators": [ + { + "firstName": "Leonard Noel Mansell", + "lastName": "Newell", + "creatorType": "author" + } + ], + "date": "1938", + "callNumber": "R003853768", + "language": "epo", + "libraryCatalog": "Library Catalog (RERO ILS)", + "numPages": "149", + "place": "Budapest", + "publisher": "Literatura Mondo", + "shortTitle": "Bakŝis, skizoj el la vivo de Egiptoj", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/Library of Congress ISBN.js b/Library of Congress ISBN.js index 7544f17feee..9978d44195b 100644 --- a/Library of Congress ISBN.js +++ b/Library of Congress ISBN.js @@ -8,11 +8,9 @@ "priority": 97, "inRepository": true, "translatorType": 8, - "browserSupport": "gcsibv", - "lastUpdated": "2022-02-07 18:32:41" + "lastUpdated": "2023-08-25 05:06:07" } - function detectSearch(item) { //re-enable once /*if (item.ISBN) { @@ -23,15 +21,16 @@ function detectSearch(item) { function doSearch(item) { - //Sends an SRU formatted as CQL to the library of Congress asking for marcXML back - //http://www.loc.gov/standards/sru/ + // Sends an SRU formatted as CQL to the Library of Congress asking for MARCXML back + // https://www.loc.gov/standards/sru/ + // https://www.loc.gov/z3950/lcserver.html let url; if (item.ISBN) { - url = "http://lx2.loc.gov:210/LCDB?operation=searchRetrieve&version=1.1&query=bath.ISBN=^" + ZU.cleanISBN(item.ISBN) + "&maximumRecords=1"; + url = "https://lx2.loc.gov/sru/lcdb?operation=searchRetrieve&version=1.1&query=bath.ISBN=^" + ZU.cleanISBN(item.ISBN) + "&maximumRecords=1"; } else if (item.query) { - url = "http://lx2.loc.gov:210/LCDB?operation=searchRetrieve&version=1.1&query=" + encodeURIComponent(item.query) + "&maximumRecords=50"; + url = "https://lx2.loc.gov/sru/lcdb?operation=searchRetrieve&version=1.1&query=" + encodeURIComponent(item.query) + "&maximumRecords=50"; } ZU.doGet(url, function (text) { diff --git a/Literary Hub.js b/Literary Hub.js new file mode 100644 index 00000000000..d918ff844f3 --- /dev/null +++ b/Literary Hub.js @@ -0,0 +1,950 @@ +{ + "translatorID": "f1aaf4bf-f344-41ee-b960-905255d40d53", + "label": "Literary Hub", + "creator": "Zoë C. Ma", + "target": "^https?://lithub\\.com/", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-04-22 01:23:39" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Zoë C. Ma + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see <http://www.gnu.org/licenses/>. + + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { + const urlObj = new URL(url); + + // Reject "non-processable" pages, i.e. pages for site-structural + // purposes, including legal notices, about pages, taxonomy, etc. + // Not incorporating this logic into the "target" field of translator + // metadata, to keep the RegExp there simple. + const skipPath = /\/(category|tag|masthead|about-literary-hub|privacy-policy-for-.+)\//; + if (urlObj.pathname.match(skipPath)) { + return false; + } + + const searchValue = urlObj.searchParams.get("s"); + if (urlObj.pathname === "/") { // On search page (i.e. site base URL). + if (!searchValue) { + // Empty search (searchValue is empty string) or a page + // on the expected path but without the required + // search-query parameter; skip content check and + // reject directly. + return false; + } + if (getSearchResults(doc, true)) { + return "multiple"; + } + else { + return false; + } + } + // Not on search page. + if (doc.querySelector("div.post_wrapper")) { + const tags = getTags(doc); + if (tags && tags.includes("podcasts")) { + return "podcast"; + } + return "blogPost"; + } + return false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) === "multiple") { + const items = await Z.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (const url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +function getSearchResults(doc, checkOnly) { + const resultElems = doc.querySelectorAll("div.search .post_header a"); + if (!resultElems.length) return false; + + const items = {}; + let isNonEmpty = false; + for (const anchorElem of resultElems) { + const href = anchorElem.href; + const title = ZU.trimInternal(anchorElem.textContent.trim()); + + // Currently, external hyperlinks in search results + // will not work correctly, because they are passed to + // the same scrape() function that is not designed for + // them. Therefore, they are skipped. + const hrefURL = new URL(href); + if (hrefURL.origin === doc.location.origin + && !(href in items) && title) { + if (checkOnly) { + return true; + } + items[href] = title; + isNonEmpty = true; + } // Otherwise skip duplicate or external result. + } + return isNonEmpty && items; +} + +async function scrape(doc, url) { // eslint-disable-line no-unused-vars + const translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + translator.setHandler('itemDone', function (tobj, item) { + handleItem(doc, item); + }); + translator.translate(); +} + +// Detect the LitHub article type (plain, Lit Hub Excerpts, ...). +// NOTE: Must be called after the item's tags have been populated, because some +// detection routines require tag data. +function detectType(doc) { + if (doc.querySelector("div.excerptpage")) { + return "excerpt"; + } + // Other types of posts, if needed, may be add here. + return false; +} + +// Handle generic item by populating the tags and publicationTitle fields and +// dispatch the handler based on the document's type. +function handleItem(doc, item) { + item.publicationTitle = "Literary Hub"; + + const tags = getTags(doc); + item.tags = tags; + + const articleType = tags.map(t => t.toLowerCase()).includes("podcasts") + ? "podcast" + : detectType(doc); + + // Populate the fields (especially authorship info) based on the type + // of the article. + switch (articleType) { + case "excerpt": + handleExcerpt(doc, item); + break; + case "podcast": + handlePodcast(doc, item); + break; + default: + handleDefault(doc, item); + } + + item.complete(); +} + +// Type-specific handlers. + +// Podcasts hosted by LitHub. +function handlePodcast(doc, item) { + item.itemType = "podcast"; + item.seriesTitle = getByline(doc); + item.creators = getPodcastAuthors(doc); + item.creators.push(...inferPodcastGuests(doc, item.title)); +} + +function getPodcastAuthors(doc) { + const textLine = getSubheading(doc); + if (!textLine) return []; + + let rawAuth = textLine.match(/(?:conversation with\s+)(.+)(?:\s+on\s+)/i); + if (!rawAuth) { + rawAuth = textLine.match(/(?:Hosted by\s+)(.+)/i); + } + if (rawAuth) { + return splitNames(rawAuth[1]) + .map(n => ZU.cleanAuthor(n, "author")); + } + return []; +} + +// Infer the podcast guests from the body paragraphs. The opening bold text of +// paragraphs are where the names can usually be found. To filter out noise, +// notice that the guest's name is also most likely part of the title. +// NOTE: an alternative idea is to look for the guest's name in the tags. +function inferPodcastGuests(doc, title) { + const podBody = doc.querySelector("[itemprop='articleBody']"); + if (!podBody) return false; + + const normTitle = title.toLowerCase(); + + const boldFrags = []; + for (const elem of podBody.querySelectorAll(":scope p b, :scope p strong")) { + const txtMatch = ZU.trimInternal(elem.textContent).match(/\b.+\b/); + if (txtMatch) { + const txt = txtMatch[0]; + if (txt === txt.toUpperCase()) { + // Skip all-caps, which are relatively noisy. + continue; + } + if (!boldFrags.includes(txt) + && normTitle.includes(txt.toLowerCase())) { + boldFrags.push(txt); + } + } + } + return boldFrags.map(name => ZU.cleanAuthor(name, "guest")); +} + +// Default, or "plain" blog post. +function handleDefault(doc, item) { + const creators = []; + const rawAuthors = getByline(doc); + // Skip "institutional author" when it is the same as publisher. + if (rawAuthors && rawAuthors.toLowerCase() !== "literary hub") { + const authorInfo = parseAuthorTransFromDefault(rawAuthors, doc, item); + if (authorInfo) { + for (const [type, names] of Object.entries(authorInfo)) { + creators.push(...names.map(n => ZU.cleanAuthor(n, type))); + } + } + } + item.creators = creators; +} + +// Given the document of the article of the "default" type, determine author +// and possible translator information as an object with properties "author" +// and "translator", or false if not found. +function parseAuthorTransFromDefault(bylineText, doc, item) { + // Attempt to parse the raw-byline as "...[,] translated by [...]" or + // similar. + const bylineMatch = bylineText.match(/(.+?)[,;\s([]+trans(?:lated\s+by\s+|\.\s+)(.+)\b/i); + if (bylineMatch) { + return { + author: splitNames(bylineMatch[1]), + translator: splitNames(bylineMatch[2]) + }; + } + + // Attempt to parse the byline without any "translated by" or "trans." + // hints in it. To check for any translator(s), check for any that is + // included in the title or subtitle after "translated by ...". + const transHeadingRE = /\btranslated by\s+(.+)/i; + // Check title. + let transHeadingMatch = item.title.match(transHeadingRE); + if (!transHeadingMatch) { + // Check subtitle h3. + const subheading = getSubheading(doc); + transHeadingMatch = subheading.match(transHeadingRE); + } + if (transHeadingMatch) { + const translators = splitNames(transHeadingMatch[1]); + // Dedup any byline names. + const authors = []; + for (const name of splitNames(bylineText)) { + if (!translators.includes(name)) { + authors.push(name); + } + } + return { + author: authors, + translator: translators + }; + } + + // Just author names, without translators. + return { + author: splitNames(bylineText), + translator: [] + }; +} + +// Handle the page that belongs to the "Lit Hub Excerpts" type. +function handleExcerpt(doc, item) { + const rawAuthorText = ZU.trimInternal(text(doc, "div.excerptpage>h2")); + // Split into possible main author(s)/translator(s) parts. + const authTransMatch = rawAuthorText.match(/(.+)\s*\(trans\.\s(.+)\)/); + + const creators = []; + let nameText; + if (authTransMatch) { + // Translated work. + nameText = authTransMatch[2]; // translator(s) + for (const name of splitNames(nameText)) { + creators.push(ZU.cleanAuthor(name, "translator")); + } + nameText = authTransMatch[1]; // original author(s) + } + else { + nameText = rawAuthorText; + } + + // Handle original author name(s). + for (const name of splitNames(nameText)) { + creators.push(ZU.cleanAuthor(name, "author")); + } + + item.creators = creators; +} + +// Convenience utility functions + +// Split a string of personal names delimitered by the comma (,) and optionally +// with the word "and", no matter whether the Oxford comma style is used. +function splitNames(str) { + return str.split(/(?:,|\s+and\s+)/) + .map(s => s.trim()) + .filter(Boolean); +} + +// Returns an array of tags on the article. +function getTags(doc) { + return Array.from(doc.querySelectorAll(".post_tag a[rel='tag']")) + .map(elem => ZU.trimInternal(elem.textContent.trim())); +} + +// Returns the (whitespace-normalized) byline ("By ...") text. +function getByline(doc) { + let byline = text(doc, ".author_name span[itemprop='name']"); + if (!byline) { + // Only for certain pre-2022 posts without more + // semantically-clear byline info. + byline = text(doc, ".post_detail > a[href*='author']"); + } + return ZU.trimInternal(byline); +} + +// Returns the h3 subheading in the post heading block, which can be rich in +// information. +function getSubheading(doc) { + return ZU.trimInternal(text(doc, ".post_header_wrapper h3")); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://lithub.com/?s=wonder+AND+lelio", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://lithub.com/lydia-conklin-on-letting-their-personality-into-their-work/", + "detectedItemType": "podcast", + "items": [ + { + "itemType": "podcast", + "title": "Lydia Conklin on Letting Their Personality Into Their Work", + "creators": [ + { + "firstName": "Alex", + "lastName": "Higley", + "creatorType": "author" + }, + { + "firstName": "Lindsay", + "lastName": "Hunter", + "creatorType": "author" + }, + { + "firstName": "Lydia", + "lastName": "Conklin", + "creatorType": "guest" + } + ], + "abstractNote": "Welcome to I’m a Writer But, where two writers-and talk to other writers-and about their work, their lives, their other work, the stuff that takes up any free time they have, all the stuff th…", + "language": "en-US", + "seriesTitle": "I'm a Writer But", + "url": "https://lithub.com/lydia-conklin-on-letting-their-personality-into-their-work/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Alex Higley" + }, + { + "tag": "I'm a Writer But" + }, + { + "tag": "Lindsay Hunter" + }, + { + "tag": "Lit Hub Radio" + }, + { + "tag": "Lydia Conklin" + }, + { + "tag": "Rainbow Rainbow" + }, + { + "tag": "podcasts" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/the-cat-thief-by-son-bo-mi-translated-by-janet-hong/", + "detectedItemType": "blogPost", + "items": [ + { + "itemType": "blogPost", + "title": "“The Cat Thief” by Son Bo-mi, Translated by Janet Hong", + "creators": [ + { + "firstName": "Son", + "lastName": "Bo-mi", + "creatorType": "author" + }, + { + "firstName": "Janet", + "lastName": "Hong", + "creatorType": "translator" + } + ], + "date": "2022-11-08T09:51:52+00:00", + "abstractNote": "“I was away from Korea for a long time,” he said. We were having tea at a downtown café. I tried to recall the last time I’d seen him, but couldn’t. When I made some offhand comment about the tea t…", + "blogTitle": "Literary Hub", + "language": "en-US", + "url": "https://lithub.com/the-cat-thief-by-son-bo-mi-translated-by-janet-hong/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Freeman's" + }, + { + "tag": "Janet Hong" + }, + { + "tag": "Korean literature" + }, + { + "tag": "Son Bo-mi" + }, + { + "tag": "The Cat Thief" + }, + { + "tag": "loss" + }, + { + "tag": "theft" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/read-the-winners-of-american-short-fictions-2022-insider-prize-selected-by-lauren-hough/", + "detectedItemType": "blogPost", + "items": [ + { + "itemType": "blogPost", + "title": "Read the Winners of American Short Fiction’s 2022 Insider Prize, Selected by Lauren Hough", + "creators": [], + "date": "2022-09-15T08:55:13+00:00", + "abstractNote": "Lauren Hough may be known for her spar-ready online presence, but in real life she’s pure warmth: years ago, she overheard us talking about the Insider Prize—American Short Fiction’s annual contest…", + "blogTitle": "Literary Hub", + "language": "en-US", + "url": "https://lithub.com/read-the-winners-of-american-short-fictions-2022-insider-prize-selected-by-lauren-hough/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "American Short Fiction" + }, + { + "tag": "David Antares" + }, + { + "tag": "Insider Prize" + }, + { + "tag": "Lauren Hough" + }, + { + "tag": "Michael John Wiese" + }, + { + "tag": "incarcerated writers" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/eden-boudreau-on-memoirs-that-risk-everything/", + "detectedItemType": "podcast", + "items": [ + { + "itemType": "podcast", + "title": "Eden Boudreau on Memoirs That Risk Everything", + "creators": [ + { + "firstName": "Brooke", + "lastName": "Warner", + "creatorType": "author" + }, + { + "firstName": "Grant", + "lastName": "Faulkner", + "creatorType": "author" + }, + { + "firstName": "Eden", + "lastName": "Boudreau", + "creatorType": "guest" + } + ], + "abstractNote": "Write-minded: Weekly Inspiration for Writers is currently in its fourth year. We are a weekly podcast for writers craving a unique blend of inspiration and real talk about the ups and downs of the …", + "language": "en-US", + "seriesTitle": "Write-minded", + "url": "https://lithub.com/eden-boudreau-on-memoirs-that-risk-everything/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Brooke Warner" + }, + { + "tag": "Crying Wolf" + }, + { + "tag": "Eden Boudreau" + }, + { + "tag": "Grant Faulkner" + }, + { + "tag": "Lit Hub Radio" + }, + { + "tag": "Write-minded" + }, + { + "tag": "podcasts" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/lesser-islands/", + "detectedItemType": "blogPost", + "items": [ + { + "itemType": "blogPost", + "title": "Lesser Islands", + "creators": [ + { + "firstName": "Peter", + "lastName": "DiGiovanni", + "creatorType": "translator" + }, + { + "firstName": "Donatella", + "lastName": "Melucci", + "creatorType": "translator" + }, + { + "firstName": "Lorenza", + "lastName": "Pieri", + "creatorType": "author" + } + ], + "date": "2023-02-24T09:52:10+00:00", + "abstractNote": "We saw the dolphins in the morning. We trailed their shiny fins in the boat for a good half hour; then they went too far, and Papa had to turn back. For me, it was the first time. It was the end of…", + "blogTitle": "Literary Hub", + "language": "en-US", + "url": "https://lithub.com/lesser-islands/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Donatella Melucci" + }, + { + "tag": "Europa" + }, + { + "tag": "Excerpt" + }, + { + "tag": "Fiction" + }, + { + "tag": "Italian" + }, + { + "tag": "Lesser Islands" + }, + { + "tag": "Peter DiGiovanni" + }, + { + "tag": "novel" + }, + { + "tag": "translation" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/the-forest-a-fable-of-america-in-the-1830s/", + "detectedItemType": "blogPost", + "items": [ + { + "itemType": "blogPost", + "title": "The Forest: A Fable of America in the 1830s", + "creators": [ + { + "firstName": "Alexander", + "lastName": "Nemerov", + "creatorType": "author" + } + ], + "date": "2023-03-10T09:08:09+00:00", + "abstractNote": "“Smoke and Burnt Pine” Nat Turner saw hieroglyphic characters on the leaves. Written in blood, they portrayed men in different poses, the same he had seen in the sky when the Holy Ghost…", + "blogTitle": "Literary Hub", + "language": "en-US", + "shortTitle": "The Forest", + "url": "https://lithub.com/the-forest-a-fable-of-america-in-the-1830s/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Alexander Nemerov" + }, + { + "tag": "Excerpt" + }, + { + "tag": "Princeton University Press" + }, + { + "tag": "The Forest" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/a-girl-and-the-moon/", + "detectedItemType": "blogPost", + "items": [ + { + "itemType": "blogPost", + "title": "“A Girl and the Moon”", + "creators": [ + { + "firstName": "Lee", + "lastName": "Young-Ju", + "creatorType": "author" + }, + { + "firstName": "Jae", + "lastName": "Kim", + "creatorType": "translator" + } + ], + "date": "2021-12-17T09:48:57+00:00", + "abstractNote": "Mid-night, swinging upside down on a pull-up bar, the girl says, Mother, this bone growing on my back, white in the night, protruding out of my skin, long and endlessly this bone, like a ladder it …", + "blogTitle": "Literary Hub", + "language": "en-US", + "url": "https://lithub.com/a-girl-and-the-moon/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "A Girl and the Moon" + }, + { + "tag": "Black Ocean" + }, + { + "tag": "Cold Candies" + }, + { + "tag": "Jae Kim" + }, + { + "tag": "Lee Young-Ju" + }, + { + "tag": "translated poetry" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/the-calling-of-st-mattew/", + "detectedItemType": "blogPost", + "items": [ + { + "itemType": "blogPost", + "title": "“The Calling of St. Matthew”", + "creators": [ + { + "firstName": "Adam", + "lastName": "Zagajewski", + "creatorType": "author" + }, + { + "firstName": "Clare", + "lastName": "Cavanagh", + "creatorType": "translator" + } + ], + "date": "2021-12-09T09:49:32+00:00", + "abstractNote": "that priest looks just like Belmondo –Wislawa  Szymborska, Funeral (II) —Look at his hand, his palm. Like a pianist’s —But that old guy can’t see a thing —What next, paying in a church —Mom, my hea…", + "blogTitle": "Literary Hub", + "language": "en-US", + "url": "https://lithub.com/the-calling-of-st-mattew/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Adam Zagajewski" + }, + { + "tag": "Clare Cavanagh" + }, + { + "tag": "Farrar Straus and Giroux" + }, + { + "tag": "Jonathan Galassi" + }, + { + "tag": "Robyn Creswell" + }, + { + "tag": "The FSG Poetry Anthology" + }, + { + "tag": "a poem" + }, + { + "tag": "poetry" + }, + { + "tag": "translated poetry" + }, + { + "tag": "translation" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/small-wonder-the-challenge-of-parenting-through-climate-collapse/", + "detectedItemType": "blogPost", + "items": [ + { + "itemType": "blogPost", + "title": "Small Wonder: The Challenge of Parenting Through Climate Collapse", + "creators": [ + { + "firstName": "Eiren", + "lastName": "Caffall", + "creatorType": "author" + } + ], + "date": "2019-11-18T09:55:49+00:00", + "abstractNote": "We come to Monhegan Island—twelve miles off the coast of Maine, perched in the fastest-warming saltwater ecosystem on the planet—to avoid darkness. We come for wonder. We walk across the island onc…", + "blogTitle": "Literary Hub", + "language": "en-US", + "shortTitle": "Small Wonder", + "url": "https://lithub.com/small-wonder-the-challenge-of-parenting-through-climate-collapse/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Coast" + }, + { + "tag": "Environment" + }, + { + "tag": "Maine" + }, + { + "tag": "Monhegan Island" + }, + { + "tag": "Periwinkles" + }, + { + "tag": "Rachel Carson" + }, + { + "tag": "Southport" + }, + { + "tag": "The Sense of Wonder" + }, + { + "tag": "climate change" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://lithub.com/letters-to-a-writer-of-color-deepa-anappara-and-taymour-soomro-on-finding-community-with-each-other/", + "detectedItemType": "podcast", + "items": [ + { + "itemType": "podcast", + "title": "Letters to a Writer of Color: Deepa Anappara and Taymour Soomro on Finding Community With Each Other", + "creators": [ + { + "firstName": "Whitney", + "lastName": "Terrell", + "creatorType": "author" + }, + { + "firstName": "V. V.", + "lastName": "Ganeshananthan", + "creatorType": "author" + }, + { + "firstName": "Deepa", + "lastName": "Anappara", + "creatorType": "guest" + }, + { + "firstName": "Taymour", + "lastName": "Soomro", + "creatorType": "guest" + } + ], + "abstractNote": "Fiction writers Deepa Anappara and Taymour Soomro join co-hosts V.V. Ganeshananthan and Whitney Terrell to discuss the newly published essay collection Letters to a Writer of Color, which they co-e…", + "language": "en-US", + "seriesTitle": "Fiction Non Fiction", + "shortTitle": "Letters to a Writer of Color", + "url": "https://lithub.com/letters-to-a-writer-of-color-deepa-anappara-and-taymour-soomro-on-finding-community-with-each-other/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [ + { + "tag": "Deepa Anappara" + }, + { + "tag": "Fiction/Non/Fiction" + }, + { + "tag": "Letters to a Writer of Color" + }, + { + "tag": "Lit Hub Radio" + }, + { + "tag": "Taymour Soomro" + }, + { + "tag": "podcasts" + } + ], + "notes": [], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/NRC.nl.js b/NRC.nl.js new file mode 100644 index 00000000000..f2327a1e7e9 --- /dev/null +++ b/NRC.nl.js @@ -0,0 +1,226 @@ +{ + "translatorID": "8e708ceb-ce42-446f-bffd-21e2bdeb0742", + "label": "NRC.nl", + "creator": "Martijn Staal", + "target": "^https?://www\\.nrc\\.nl", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-06-20 08:03:28" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Martijn Staal + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see <http://www.gnu.org/licenses/>. + + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { + if (url.includes('/nieuws/')) { + return "newspaperArticle"; + } + else if ((url.endsWith(".nl/") || url.endsWith(".nl") || url.includes("/index/") || url.includes("/search/")) && getSearchResults(doc, true)) { + return "multiple"; + } + + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll("a.nmt-item__link"); + var isSearchPage = false; + + if (rows.length == 0) { + // We are probably not in an index page, but in a search results page. + rows = doc.querySelectorAll("div.search-results__item"); + isSearchPage = true; + } + + for (let row of rows) { + var href; + var title; + if (isSearchPage) { + href = row.getAttribute("data-article-url"); + title = row.getAttribute("data-headline"); + } + else { + href = row.href; + + if (row.textContent.includes(">")) { + title = ZU.trimInternal(row.textContent.split(">")[1]); + } + else { + title = ZU.trimInternal(row.textContent); + } + } + + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == "multiple") { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrape(doc, url = doc.location.href) { + var translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + + translator.setHandler('itemDone', function (obj, item) { + item.language = "nl-NL"; + var authors = Array.from(doc.querySelectorAll("a[rel='author']")).map(a => a.textContent); + item.creators = authors.map(a => ZU.cleanAuthor(a, 'author', false)); + + item.complete(); + }); + + translator.getTranslatorObject(function (trans) { + trans.itemType = "newspaperArticle"; + trans.doWeb(doc, url); + }); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://www.nrc.nl/nieuws/2023/06/15/lot-van-natuurherstelwet-timmermans-blijft-erg-onzeker-na-belangrijke-stemming-a4167267", + "detectedItemType": "newspaperArticle", + "items": [ + { + "itemType": "newspaperArticle", + "title": "Lot van natuurherstelwet Timmermans blijft erg onzeker na belangrijke stemming", + "creators": [ + { + "firstName": "Clara van de", + "lastName": "Wiel", + "creatorType": "author" + } + ], + "date": "2023-06-15", + "abstractNote": "Natuurherstel: Een aanzienlijke groep in het Europees Parlement is bereid om natuurplannen van Frans Timmermans de nek om te draaien, zo bleek donderdag.", + "language": "nl-NL", + "libraryCatalog": "www.nrc.nl", + "publicationTitle": "NRC", + "rights": "Copyright Mediahuis NRC BV", + "url": "https://www.nrc.nl/nieuws/2023/06/15/lot-van-natuurherstelwet-timmermans-blijft-erg-onzeker-na-belangrijke-stemming-a4167267", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.nrc.nl/nieuws/1987/06/13/reagan-vraagt-russen-de-muur-af-te-breken-kb_000035184-a3528803?t=1686835084", + "detectedItemType": "newspaperArticle", + "items": [ + { + "itemType": "newspaperArticle", + "title": "Reagan vraagt Russen de Muur af te breken", + "creators": [], + "date": "1987-06-13", + "abstractNote": "Door onze correspondent ROB MEINES BERLIJN, 13 juni - In zijn toespraak aan de Berlijnse Muur bij de Brandenburger Poort heeft president Ronald Reagan gisteren Sovjet-leider…", + "language": "nl-NL", + "libraryCatalog": "www.nrc.nl", + "publicationTitle": "NRC", + "rights": "Copyright Mediahuis NRC BV", + "url": "https://www.nrc.nl/nieuws/1987/06/13/reagan-vraagt-russen-de-muur-af-te-breken-kb_000035184-a3528803", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.nrc.nl/nieuws/2022/12/03/wikipedia-wordt-onbetrouwbaar-alweer-a4150299", + "detectedItemType": "newspaperArticle", + "items": [ + { + "itemType": "newspaperArticle", + "title": "Column | Wikipedia wordt onbetrouwbaar. Alweer", + "creators": [ + { + "firstName": "Maxim", + "lastName": "Februari", + "creatorType": "author" + } + ], + "date": "2022-12-03", + "abstractNote": "Column:Maxim Februari", + "language": "nl-NL", + "libraryCatalog": "www.nrc.nl", + "publicationTitle": "NRC", + "rights": "Copyright Mediahuis NRC BV", + "url": "https://www.nrc.nl/nieuws/2022/12/03/wikipedia-wordt-onbetrouwbaar-alweer-a4150299", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.nrc.nl/index/economie/", + "detectedItemType": "multiple", + "items": "multiple" + } +] +/** END TEST CASES **/ diff --git a/National Library of Poland ISBN.js b/National Library of Poland ISBN.js index 8789509bad4..f0141535c87 100644 --- a/National Library of Poland ISBN.js +++ b/National Library of Poland ISBN.js @@ -8,8 +8,7 @@ "priority": 98, "inRepository": true, "translatorType": 8, - "browserSupport": "gcsbv", - "lastUpdated": "2023-05-20 15:41:08" + "lastUpdated": "2023-06-15 02:45:16" } /* @@ -77,7 +76,7 @@ function doSearch(item) { // <record xmlns='http://www.loc.gov/MARC21/slim'>...</record> // </collection> // </resp> - const doc = ((new DOMParser()).parseFromString(xmlText, 'text/xml')) + const doc = ((new DOMParser()).parseFromString(xmlText, 'text/xml')); let record = doc.querySelector('collection > record'); if (!record) { return; diff --git a/News Corp Australia.js b/News Corp Australia.js index ad5a50fb009..c05d5feb390 100644 --- a/News Corp Australia.js +++ b/News Corp Australia.js @@ -9,13 +9,13 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-06-01 16:55:37" + "lastUpdated": "2023-07-14 10:46:05" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2021 Abe Jellinek + Copyright © 2021 Abe Jellinek and contributors This file is part of Zotero. @@ -78,17 +78,24 @@ function scrape(doc, url) { let meta = JSON.parse(text(doc, jsonSelector)); let item = new Zotero.Item('newspaperArticle'); - item.title = meta.headline; + // Prefer the title on the page over metadata, because the latter may be a + // different title made for social media / SEO + item.title = text(doc, "#story-headline, .story-headline__title, .long-headline") || meta.headline; item.date = ZU.strToISO(meta.datePublished); item.abstractNote = meta.description; - if (meta.author) { - item.creators = meta.author.name - .split(/ and |, /) - .map(name => ZU.cleanAuthor(name, 'author', false)); - } - item.publicationTitle = meta.publisher.name; + item.creators = getAuthors(meta.author, doc); + // Get rid of trailing strings inserted after publisher name for SEO + // purposes + item.publicationTitle = meta.publisher.name.match(/^(.+?)(?:[—|].*)?$/)[1].trim(); item.section = getSection(doc); - item.url = meta.mainEntityOfPage['@id']; + if (meta.mainEntityOfPage["@type"] === "WebPage" + && /^https?:\/\//.test(meta.mainEntityOfPage["@id"])) { + // Page URL from JSON-LD should be less prone to line noise + item.url = meta.mainEntityOfPage["@id"]; + } + if (!item.url) { + item.url = url; + } item.libraryCatalog = ''; item.attachments.push({ title: "Snapshot", @@ -99,14 +106,42 @@ function scrape(doc, url) { item.complete(); } +function getAuthors(authorFromJSON, doc) { + let byline = authorFromJSON; + if (byline) { + // The author field may be an array (multiple authors) or an atom + // object (one author) + if (!Array.isArray(byline)) { + byline = [byline]; + } + } + else { + // Fallback to scraping + byline = doc.querySelectorAll("meta[name='article:author']"); + if (!byline) { + return null; + } + byline = Array.from(byline); + } + // The "author" field may be malformed (not an array, but the "name" field + // is a string with separators between author names). Deal with "," and + // "and" as separator. + return byline.flatMap(obj => obj.name + .split(/, |and /i) + .map(str => str.trim()) + .filter(Boolean) + .map(str => ZU.cleanAuthor(str, "author", false)) + ); +} + function getSection(doc) { - let breadcrumbs = doc.querySelector('#breadcrumbs'); - if (!breadcrumbs || breadcrumbs.childElementCount < 2) { + let breadCrumbItems = doc.querySelectorAll(".breadcrumbs__item, .breadcrumbs_li"); + if (breadCrumbItems.length < 2) { // we want to return null, not '', if we can't find a section tag return text(doc, '.tg-tlc-storyheader_sectiontag') || null; } - return ZU.capitalizeTitle(breadcrumbs.lastChild.innerText, true); + return ZU.capitalizeTitle(breadCrumbItems[breadCrumbItems.length - 1].innerText, true); } /** BEGIN TEST CASES **/ diff --git a/Open WorldCat.js b/Open WorldCat.js index 8b2b27215e3..889e500f8aa 100644 --- a/Open WorldCat.js +++ b/Open WorldCat.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 12, "browserSupport": "gcsibv", - "lastUpdated": "2022-11-01 19:40:33" + "lastUpdated": "2023-08-22 04:39:47" } /* @@ -169,11 +169,9 @@ function getSearchResults(doc, checkOnly) { async function doWeb(doc, url) { if (detectWeb(doc, url) == 'multiple') { let items = await Zotero.selectItems(getSearchResults(doc, false)); - if (items) { - await Promise.all( - Object.keys(items) - .map(url => requestDocument(url).then(scrape)) - ); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } } else { @@ -269,7 +267,7 @@ function detectSearch(items) { return sanitizeInput(items, true); } -function doSearch(items) { +async function doSearch(items) { items = sanitizeInput(items); if (!items.length) { Z.debug("Search query does not contain valid identifiers"); @@ -285,72 +283,81 @@ function doSearch(items) { isbns.push(items[i].ISBN); } - - ZU.processDocuments('https://www.worldcat.org/', function (homepageDoc) { - let buildID = JSON.parse(text(homepageDoc, '#__NEXT_DATA__')).buildId; - ZU.doGet(`https://www.worldcat.org/_next/data/${buildID}/en/search.json`, function (jsonText) { - let json = JSON.parse(jsonText); - let { secureToken } = json.pageProps; - let cookie = `wc_tkn=${encodeURIComponent(secureToken)}`; - fetchIDs(isbns, ids, cookie, function (ids) { - if (!ids.length) { - Z.debug("Could not retrieve any OCLC IDs"); - Zotero.done(false); - return; - } - var url = "https://www.worldcat.org/api/search?q=no%3A" - + ids.map(encodeURIComponent).join('+OR+no%3A'); - ZU.doGet(url, function (respText) { - let json = JSON.parse(respText); - if (json.briefRecords) { - scrapeRecord(json.briefRecords); - } - }, null, null, { - Referer: 'https://worldcat.org/search?q=', - Cookie: cookie - }); - }); - }); + + let secureToken = await getSecureToken(); + let cookie = `wc_tkn=${encodeURIComponent(secureToken)}`; + ids = await fetchIDs(isbns, ids, cookie); + if (!ids.length) { + Z.debug("Could not retrieve any OCLC IDs"); + Zotero.done(false); + return; + } + var url = "https://www.worldcat.org/api/search?q=no%3A" + + ids.map(encodeURIComponent).join('+OR+no%3A'); + let json = await requestJSON(url, { + headers: { + Referer: 'https://worldcat.org/search?q=', + Cookie: cookie + } }); + if (json.briefRecords) { + scrapeRecord(json.briefRecords); + } } -function fetchIDs(isbns, ids, cookie, callback) { - if (!isbns.length) { - callback(ids); - return; +async function getSecureToken() { + let doc; + try { + doc = await requestDocument('https://www.worldcat.org/'); } - - var isbn = isbns.shift(); - // As of 10/19/2022, WorldCat's API seems to do an unindexed lookup for ISBNs without hyphens, - // with requests taking 40+ seconds, so we hyphenate them - isbn = wcHyphenateISBN(isbn); - var url = "https://www.worldcat.org/api/search?q=bn%3A" - + encodeURIComponent(isbn); - ZU.doGet(url, - function (respText) { - let json = JSON.parse(respText); - if (json.briefRecords && json.briefRecords.length) { - scrapeRecord([json.briefRecords[0]]); + catch (e) { + Z.debug('Initial request to homepage failed; trying latest Google cache'); + try { + doc = await requestDocument('https://webcache.googleusercontent.com/search?q=cache%3Aworldcat.org'); + } + catch (e) { + Z.debug('Request to Google cache failed; trying archive.org'); + doc = await requestDocument('https://web.archive.org/web/https://worldcat.org/'); + } + } + let buildID = JSON.parse(text(doc, '#__NEXT_DATA__')).buildId; + Z.debug('buildID: ' + buildID); + let json = await requestJSON(`https://www.worldcat.org/_next/data/${buildID}/en/search.json`); + let { secureToken } = json.pageProps; + Z.debug('secureToken: ' + secureToken); + return secureToken; +} + +async function fetchIDs(isbns, ids, cookie) { + while (isbns.length) { + let isbn = isbns.shift(); + // As of 10/19/2022, WorldCat's API seems to do an unindexed lookup for ISBNs without hyphens, + // with requests taking 40+ seconds, so we hyphenate them + isbn = wcHyphenateISBN(isbn); + let url = "https://www.worldcat.org/api/search?q=bn%3A" + + encodeURIComponent(isbn); + let json = await requestJSON(url, { + headers: { + Referer: 'https://worldcat.org/search?q=', + Cookie: cookie } - }, - function () { - fetchIDs(isbns, ids, cookie, callback); - }, - null, - { - Referer: 'https://worldcat.org/search?q=', - Cookie: cookie + }); + if (json.briefRecords && json.briefRecords.length) { + scrapeRecord([json.briefRecords[0]]); } - ); + } + return ids; } // Copied from Zotero.Utilities.Internal.hyphenateISBN() function wcHyphenateISBN(isbn) { // Copied from isbn.js var ISBN = {}; - ISBN.ranges = (function() { + ISBN.ranges = (function () { + /* eslint-disable */ var ranges = {"978":{"0":["00","19","200","699","7000","8499","85000","89999","900000","949999","9500000","9999999"],"1":["00","09","100","329","330","399","4000","5499","55000","86979","869800","998999","9990000","9999999"],"2":["00","19","200","349","400","699","7000","8399","35000","39999","84000","89999","900000","949999","9500000","9999999"],"3":["00","02","04","19","030","033","200","699","0340","0369","7000","8499","03700","03999","85000","89999","95400","96999","99000","99499","99500","99999","900000","949999","9500000","9539999","9700000","9899999"],"5":["01","19","200","420","430","430","440","440","450","699","0050","0099","4210","4299","4310","4399","4410","4499","7000","8499","9200","9299","9501","9799","9910","9999","00000","00499","85000","89999","91000","91999","93000","94999","98000","98999","900000","909999","9500000","9500999","9900000","9909999"],"600":["00","09","100","499","5000","8999","90000","99999"],"601":["00","19","85","99","200","699","7000","7999","80000","84999"],"602":["00","07","200","699","0800","0899","0900","1099","1100","1199","1200","1399","1500","1699","7500","7999","8000","9499","14000","14999","17000","17999","18000","18999","19000","19999","70000","74999","95000","99999"],"603":["00","04","05","49","500","799","8000","8999","90000","99999"],"604":["0","4","50","89","900","979","9800","9999"],"605":["01","02","04","09","030","039","100","399","4000","5999","9000","9999","60000","89999"],"606":["0","0","10","49","500","799","8000","9199","92000","99999"],"607":["00","39","400","749","7500","9499","95000","99999"],"608":["0","0","7","9","10","19","200","449","4500","6499","65000","69999"],"609":["00","39","400","799","8000","9499","95000","99999"],"612":["00","29","50","99","300","399","4000","4499","45000","49999"],"613":["0","9"],"615":["00","09","100","499","5000","7999","80000","89999"],"616":["00","19","200","699","7000","8999","90000","99999"],"617":["00","49","500","699","7000","8999","90000","99999"],"618":["00","19","200","499","5000","7999","80000","99999"],"619":["00","14","150","699","7000","8999","90000","99999"],"621":["00","29","400","599","8000","8999","95000","99999"],"7":["00","09","100","499","5000","7999","80000","89999","900000","999999"],"80":["00","19","200","699","7000","8499","85000","89999","900000","999999"],"82":["00","19","200","689","7000","8999","90000","98999","690000","699999","990000","999999"],"83":["00","19","200","599","7000","8499","60000","69999","85000","89999","900000","999999"],"84":["00","13","140","149","200","699","7000","8499","9000","9199","9700","9999","15000","19999","85000","89999","92400","92999","95000","96999","920000","923999","930000","949999"],"85":["00","19","200","549","5500","5999","7000","8499","60000","69999","85000","89999","98000","99999","900000","979999"],"86":["00","29","300","599","6000","7999","80000","89999","900000","999999"],"87":["00","29","400","649","7000","7999","85000","94999","970000","999999"],"88":["00","19","200","599","910","929","6000","8499","9300","9399","85000","89999","95000","99999","900000","909999","940000","949999"],"89":["00","24","250","549","990","999","5500","8499","85000","94999","97000","98999","950000","969999"],"90":["00","19","90","90","94","94","200","499","5000","6999","8500","8999","70000","79999","800000","849999"],"91":["0","1","20","49","500","649","7000","7999","85000","94999","970000","999999"],"92":["0","5","60","79","800","899","9000","9499","95000","98999","990000","999999"],"93":["00","09","100","499","5000","7999","80000","94999","950000","999999"],"94":["000","599","6000","8999","90000","99999"],"950":["00","49","500","899","9000","9899","99000","99999"],"951":["0","1","20","54","550","889","8900","9499","95000","99999"],"952":["00","19","60","65","80","94","200","499","5000","5999","6600","6699","7000","7999","9500","9899","67000","69999","99000","99999"],"953":["0","0","10","14","51","54","150","509","6000","9499","55000","59999","95000","99999"],"954":["00","28","300","799","2900","2999","8000","8999","9300","9999","90000","92999"],"955":["20","40","550","749","0000","1999","4500","4999","7500","7999","8000","9499","41000","43999","44000","44999","50000","54999","95000","99999"],"956":["00","19","200","699","7000","9999"],"957":["00","02","05","19","21","27","31","43","440","819","0300","0499","2000","2099","8200","9699","28000","30999","97000","99999"],"958":["00","56","600","799","8000","9499","57000","59999","95000","99999"],"959":["00","19","200","699","7000","8499","85000","99999"],"960":["00","19","93","93","200","659","690","699","6600","6899","7000","8499","9400","9799","85000","92999","98000","99999"],"961":["00","19","200","599","6000","8999","90000","94999"],"962":["00","19","200","699","900","999","7000","8499","8700","8999","85000","86999"],"963":["00","19","200","699","7000","8499","9000","9999","85000","89999"],"964":["00","14","150","249","300","549","970","989","2500","2999","5500","8999","9900","9999","90000","96999"],"965":["00","19","200","599","7000","7999","90000","99999"],"966":["00","12","14","14","130","139","170","199","279","289","300","699","910","949","980","999","1500","1699","2000","2789","2900","2999","7000","8999","90000","90999","95000","97999"],"967":["00","00","60","89","300","499","900","989","0100","0999","5000","5999","9900","9989","10000","19999","99900","99999"],"968":["01","39","400","499","800","899","5000","7999","9000","9999"],"969":["0","1","20","22","24","39","400","749","7500","9999","23000","23999"],"970":["01","59","600","899","9000","9099","9700","9999","91000","96999"],"971":["02","02","06","49","97","98","000","015","500","849","0160","0199","0300","0599","8500","9099","9600","9699","9900","9999","91000","95999"],"972":["0","1","20","54","550","799","8000","9499","95000","99999"],"973":["0","0","20","54","100","169","550","759","1700","1999","7600","8499","8900","9499","85000","88999","95000","99999"],"974":["00","19","200","699","7000","8499","9500","9999","85000","89999","90000","94999"],"975":["02","24","250","599","990","999","6000","9199","00000","01999","92000","98999"],"976":["0","3","40","59","600","799","8000","9499","95000","99999"],"977":["00","19","90","99","200","499","700","849","5000","6999","85000","89999"],"978":["000","199","900","999","2000","2999","8000","8999","30000","79999"],"979":["20","29","000","099","400","799","1000","1499","3000","3999","8000","9499","15000","19999","95000","99999"],"980":["00","19","200","599","6000","9999"],"981":["00","11","200","289","290","299","310","399","3000","3099","4000","9999","17000","19999"],"982":["00","09","70","89","100","699","9000","9799","98000","99999"],"983":["00","01","45","49","50","79","020","199","800","899","2000","3999","9000","9899","40000","44999","99000","99999"],"984":["00","39","400","799","8000","8999","90000","99999"],"985":["00","39","400","599","6000","8999","90000","99999"],"986":["00","11","120","559","5600","7999","80000","99999"],"987":["00","09","30","35","40","44","500","899","1000","1999","3600","3999","9000","9499","20000","29999","45000","49999","95000","99999"],"988":["00","11","200","799","8000","9699","12000","14999","15000","16999","17000","19999","97000","99999"],"9925":["0","2","30","54","550","734","7350","9999"],"9926":["0","1","20","39","400","799","8000","9999"],"9927":["00","09","100","399","4000","4999"],"9929":["0","3","40","54","550","799","8000","9999"],"9930":["00","49","500","939","9400","9999"],"9931":["00","29","300","899","9000","9999"],"9932":["00","39","400","849","8500","9999"],"9933":["0","0","10","39","400","899","9000","9999"],"9934":["0","0","10","49","500","799","8000","9999"],"9937":["0","2","30","49","500","799","8000","9999"],"9938":["00","79","800","949","9500","9999"],"9939":["0","4","50","79","800","899","9000","9999"],"9940":["0","1","20","49","500","899","9000","9999"],"9942":["00","84","900","984","8500","8999","9850","9999"],"9943":["00","29","300","399","975","999","4000","9749"],"9944":["60","69","80","89","100","499","700","799","900","999","0000","0999","5000","5999"],"9945":["00","00","08","39","57","57","010","079","400","569","580","849","8500","9999"],"9946":["0","1","20","39","400","899","9000","9999"],"9947":["0","1","20","79","800","999"],"9949":["0","0","10","39","75","89","400","749","9000","9999"],"9950":["00","29","300","849","8500","9999"],"9953":["0","0","10","39","60","89","400","599","9000","9999"],"9955":["00","39","400","929","9300","9999"],"9957":["00","39","70","84","88","99","400","699","8500","8799"],"9958":["00","01","10","18","20","49","020","029","040","089","500","899","0300","0399","0900","0999","1900","1999","9000","9999"],"9959":["0","1","20","79","98","99","800","949","970","979","9500","9699"],"9960":["00","59","600","899","9000","9999"],"9961":["0","2","30","69","700","949","9500","9999"],"9962":["00","54","56","59","600","849","5500","5599","8500","9999"],"9963":["0","1","30","54","250","279","550","734","2000","2499","2800","2999","7350","7499","7500","9999"],"9964":["0","6","70","94","950","999"],"9965":["00","39","400","899","9000","9999"],"9966":["20","69","000","149","750","959","1500","1999","7000","7499","9600","9999"],"9971":["0","5","60","89","900","989","9900","9999"],"9972":["1","1","00","09","30","59","200","249","600","899","2500","2999","9000","9999"],"9973":["00","05","10","69","060","089","700","969","0900","0999","9700","9999"],"9974":["0","2","30","54","95","99","550","749","7500","9499"],"9975":["0","0","45","89","100","299","900","949","3000","3999","4000","4499","9500","9999"],"9977":["00","89","900","989","9900","9999"],"9978":["00","29","40","94","300","399","950","989","9900","9999"],"9979":["0","4","50","64","66","75","650","659","760","899","9000","9999"],"9980":["0","3","40","89","900","989","9900","9999"],"9981":["00","09","20","79","100","159","800","949","1600","1999","9500","9999"],"9982":["00","79","800","989","9900","9999"],"9983":["80","94","950","989","9900","9999"],"9984":["00","49","500","899","9000","9999"],"9986":["00","39","97","99","400","899","940","969","9000","9399"],"9987":["00","39","400","879","8800","9999"],"9988":["0","2","30","54","550","749","7500","9999"],"9989":["0","0","30","59","100","199","600","949","2000","2999","9500","9999"],"99901":["00","49","80","99","500","799"],"99903":["0","1","20","89","900","999"],"99904":["0","5","60","89","900","999"],"99905":["0","3","40","79","800","999"],"99906":["0","2","30","59","70","89","90","94","600","699","950","999"],"99908":["0","0","10","89","900","999"],"99909":["0","3","40","94","950","999"],"99910":["0","2","30","89","900","999"],"99911":["00","59","600","999"],"99912":["0","3","60","89","400","599","900","999"],"99913":["0","2","30","35","600","604"],"99914":["0","4","50","89","900","999"],"99915":["0","4","50","79","800","999"],"99916":["0","2","30","69","700","999"],"99919":["0","2","40","69","70","79","300","399","800","849","850","899","900","999"],"99921":["0","1","8","8","20","69","90","99","700","799"],"99922":["0","3","40","69","700","999"],"99926":["0","0","10","59","87","89","90","99","600","869"],"99927":["0","2","30","59","600","999"],"99928":["0","0","10","79","800","999"],"99932":["0","0","7","7","10","59","80","99","600","699"],"99935":["0","2","7","8","30","59","90","99","600","699"],"99936":["0","0","10","59","600","999"],"99937":["0","1","20","59","600","999"],"99938":["0","1","20","59","90","99","600","899"],"99940":["0","0","10","69","700","999"],"99941":["0","2","30","79","800","999"],"99953":["0","2","30","79","94","99","800","939"],"99954":["0","2","30","69","88","99","700","879"],"99955":["0","1","20","59","80","99","600","799"],"99956":["00","59","86","99","600","859"],"99958":["0","4","50","93","940","949","950","999"],"99960":["0","0","10","94","950","999"],"99961":["0","3","40","89","900","999"],"99963":["00","49","92","99","500","919"],"99966":["0","2","30","69","80","94","700","799"],"99967":["0","1","20","59","600","899"],"99971":["0","5","60","84","850","999"],"99974":["40","79","800","999"],"99976":["0","1","20","59","600","799"]},"979":{"10":["00","19","200","699","7000","8999","90000","97599","976000","999999"],"11":["00","24","250","549","5500","8499","85000","94999","950000","999999"],"12":["200","200"]}}; ranges['978']['99968']=ranges['978']['99912'];ranges['978']['9935']=ranges['978']['9941']=ranges['978']['9956']=ranges['978']['9933'];ranges['978']['9976']=ranges['978']['9971'];ranges['978']['99949']=ranges['978']['99903'];ranges['978']['9968']=ranges['978']['9930'];ranges['978']['99929']=ranges['978']['99930']=ranges['978']['99931']=ranges['978']['99942']=ranges['978']['99944']=ranges['978']['99948']=ranges['978']['99950']=ranges['978']['99952']=ranges['978']['99962']=ranges['978']['99969']=ranges['978']['99915'];ranges['978']['99917']=ranges['978']['99910'];ranges['978']['99920']=ranges['978']['99970']=ranges['978']['99972']=ranges['978']['99914'];ranges['978']['99933']=ranges['978']['99943']=ranges['978']['99946']=ranges['978']['99959']=ranges['978']['99927'];ranges['978']['81']=ranges['978']['80'];ranges['978']['9967']=ranges['978']['9970']=ranges['978']['9965'];ranges['978']['9936']=ranges['978']['9952']=ranges['978']['9954']=ranges['978']['9926'];ranges['978']['99965']=ranges['978']['99922'];ranges['978']['9928']=ranges['978']['9927'];ranges['978']['99947']=ranges['978']['99916'];ranges['978']['9985']=ranges['978']['9939'];ranges['978']['99918']=ranges['978']['99925']=ranges['978']['99973']=ranges['978']['99975']=ranges['978']['99905'];ranges['978']['99939']=ranges['978']['99945']=ranges['978']['99904'];ranges['978']['989']=ranges['978']['972'];ranges['978']['620']=ranges['978']['613'];ranges['978']['4']=ranges['978']['0'];ranges['978']['99923']=ranges['978']['99924']=ranges['978']['99934']=ranges['978']['99957']=ranges['978']['99964']=ranges['978']['9947'];ranges['978']['614']=ranges['978']['609'];ranges['978']['9948']=ranges['978']['9951']=ranges['978']['9932']; + /* eslint-enable */ return ranges; })(); var ranges = ISBN.ranges, @@ -359,8 +366,9 @@ function wcHyphenateISBN(isbn) { i = 0; if (isbn.length == 10) { uccPref = '978'; - } else { - uccPref = isbn.substr(0,3); + } + else { + uccPref = isbn.substr(0, 3); if (!ranges[uccPref]) return ''; // Probably invalid ISBN, but the checksum is OK parts.push(uccPref); i = 3; // Skip ahead @@ -368,7 +376,7 @@ function wcHyphenateISBN(isbn) { var group = '', found = false; - while (i < isbn.length-3 /* check digit, publication, registrant */) { + while (i < isbn.length - 3 /* check digit, publication, registrant */) { group += isbn.charAt(i); if (ranges[uccPref][group]) { parts.push(group); @@ -390,12 +398,12 @@ function wcHyphenateISBN(isbn) { var registrant = ''; found = false; i++; // Previous loop 'break'ed early - while (!found && i < isbn.length-2 /* check digit, publication */) { + while (!found && i < isbn.length - 2 /* check digit, publication */) { registrant += isbn.charAt(i); - for(let j=0; j < regRanges.length && registrant.length >= regRanges[j].length; j+=2) { - if(registrant.length == regRanges[j].length - && registrant >= regRanges[j] && registrant <= regRanges[j+1] // Falls within the range + for (let j = 0; j < regRanges.length && registrant.length >= regRanges[j].length; j += 2) { + if (registrant.length == regRanges[j].length + && registrant >= regRanges[j] && registrant <= regRanges[j + 1] // Falls within the range ) { parts.push(registrant); found = true; @@ -408,8 +416,8 @@ function wcHyphenateISBN(isbn) { if (!found) return ''; // Outside of valid range, but maybe we need to update our data - parts.push(isbn.substring(i,isbn.length-1)); // Publication is the remainder up to last digit - parts.push(isbn.charAt(isbn.length-1)); // Check digit + parts.push(isbn.substring(i, isbn.length - 1)); // Publication is the remainder up to last digit + parts.push(isbn.charAt(isbn.length - 1)); // Check digit return parts.join('-'); } @@ -972,7 +980,8 @@ var testCases = [ }, { "type": "web", - "url": "http://www.worldcat.org/search?q=isbn%3A7112062314", + "url": "https://www.worldcat.org/search?q=isbn%3A7112062314", + "defer": true, "items": [ { "itemType": "book", diff --git a/Optimization Online.js b/Optimization Online.js new file mode 100644 index 00000000000..f6a92677ff7 --- /dev/null +++ b/Optimization Online.js @@ -0,0 +1,418 @@ +{ + "translatorID": "0ee623f9-0a5c-4f6a-a5e1-10e2feaa16a7", + "label": "Optimization Online", + "creator": "Pascal Quach", + "target": "^https?://optimization-online\\.org/", + "minVersion": "6.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-07-15 13:50:06" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Pascal Quach <pascal.quach@centralesupelec.fr> + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see <http://www.gnu.org/licenses/>. + + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { // eslint-disable-line no-unused-vars + if (doc.body.classList.contains('single-post')) { + return 'preprint'; + } + else if (getSearchResults(doc, true)) { + return 'multiple'; + } + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('.entry-title > a'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrape(doc, url) { + let translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + + translator.setHandler('itemDone', (_obj, item) => { + // Item + item.itemType = 'preprint'; + // Use innerText instead of textContent (duplicated LaTeX) + let title = innerText(doc, 'h1.entry-title'); + if (title) { + item.title = ZU.capitalizeTitle(title); + } + let tags = doc.querySelectorAll('div.entry-meta > span.tags-links > a[rel="tag"]'); + for (let tag of tags) { + item.tags.push(ZU.cleanTags(tag.text)); + } + // Use innerText instead of textContent (duplicated LaTeX) + let abstractNote = innerText(doc, 'div.entry-content > div > p'); + if (abstractNote) { + item.abstractNote = ZU.trimInternal(abstractNote); + } + + // Attachment + let attachmentUrl = attr(doc, '.entry-content a[href*="/wp-content/uploads/"]', 'href'); + item.attachments = []; + if (attachmentUrl.length) { + item.attachments.push({ + title: 'Full Text PDF', + mimeType: 'application/pdf', + url: attachmentUrl, + }); + } + + // Archive + item.publisher = 'Optimization Online'; // repository + item.libraryCatalog = 'optimization-online.org'; + let archiveID = attr(doc, 'article', 'id'); + if (archiveID) { + item.archiveID = archiveID.split('-')[1]; + } + let shortUrl = doc.querySelector('span.shorturl > a').href; + if (shortUrl) { + item.url = shortUrl; + } + + item.complete(); + }); + + let em = await translator.getTranslatorObject(); + await em.doWeb(doc, url); +} + + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://optimization-online.org/2023/05/political-districting-to-optimize-the-polsby-popper-compactness-score/", + "detectedItemType": "preprint", + "items": [ + { + "itemType": "preprint", + "title": "Political districting to optimize the Polsby-Popper compactness score", + "creators": [ + { + "firstName": "Pietro", + "lastName": "Belotti", + "creatorType": "author" + }, + { + "firstName": "Austin", + "lastName": "Buchanan", + "creatorType": "author" + }, + { + "firstName": "Soraya", + "lastName": "Ezazipour", + "creatorType": "author" + } + ], + "date": "2023-05-19", + "abstractNote": "In the academic literature and in expert testimony, the Polsby-Popper score is the most popular way to measure the compactness of a political district. Given a district with area A and perimeter P, its Polsby-Popper score is given by (4 \\pi A)/P^2. This score takes values between zero and one, with circular districts achieving a perfect score of one. In this paper, we propose the first mathematical optimization models to draw districts with optimum Polsby-Popper score. Specifically, we propose new mixed-integer second-order cone programs (MISOCPs), which can be solved with existing optimization software. We investigate their tractability by applying them to real-life districting instances in the USA. Experiments show that they are able to: (i) identify the most compact majority-minority districts at the tract level; (ii) identify the most compact districting plans at the county level; and (iii) refine existing tract-level plans to make them substantially more compact. Our techniques could be used by plaintiffs when seeking to overturn maps that dilute the voting strength of minority groups. Our python code and results are publicly available on GitHub.", + "archiveID": "23021", + "language": "en-US", + "libraryCatalog": "optimization-online.org", + "repository": "Optimization Online", + "url": "https://optimization-online.org/?p=23021", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "MINLP" + }, + { + "tag": "districting" + }, + { + "tag": "misocp" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://optimization-online.org/2023/06/mind-the-tildeo-asymptotically-better-but-still-impractical-quantum-distributed-algorithms/", + "detectedItemType": "preprint", + "items": [ + { + "itemType": "preprint", + "title": "Mind the \\tilde{O}: asymptotically better, but still impractical, quantum distributed algorithms", + "creators": [ + { + "firstName": "David E. Bernal", + "lastName": "Neira", + "creatorType": "author" + } + ], + "date": "2023-06-22", + "archiveID": "23283", + "language": "en-US", + "libraryCatalog": "optimization-online.org", + "repository": "Optimization Online", + "shortTitle": "Mind the \\tilde{O}", + "url": "https://optimization-online.org/?p=23283", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "Directed Minimum Spanning Tree" + }, + { + "tag": "complexity" + }, + { + "tag": "distributed computing" + }, + { + "tag": "quantum computing" + }, + { + "tag": "steiner tree" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://optimization-online.org/2023/05/maximum-likelihood-probability-measures-over-sets-and-applications-to-data-driven-optimization/", + "detectedItemType": "preprint", + "items": [ + { + "itemType": "preprint", + "title": "Maximum Likelihood Probability Measures over Sets and Applications to Data-Driven Optimization", + "creators": [ + { + "firstName": "Juan", + "lastName": "Borrero", + "creatorType": "author" + }, + { + "firstName": "Denis", + "lastName": "Saure", + "creatorType": "author" + } + ], + "date": "2023-05-15", + "archiveID": "22948", + "language": "en-US", + "libraryCatalog": "optimization-online.org", + "repository": "Optimization Online", + "url": "https://optimization-online.org/?p=22948", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "data-driven decision making" + }, + { + "tag": "distributionally robust optimization" + }, + { + "tag": "maximum likelihood estimation" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://optimization-online.org/2023/05/modeling-risk-for-cvar-based-decisions-in-risk-aggregation/", + "detectedItemType": "preprint", + "items": [ + { + "itemType": "preprint", + "title": "Modeling risk for CVaR-based decisions in risk aggregation", + "creators": [ + { + "firstName": "Yuriy", + "lastName": "Zinchenko", + "creatorType": "author" + } + ], + "date": "2023-05-08", + "abstractNote": "Download", + "archiveID": "22890", + "language": "en-US", + "libraryCatalog": "optimization-online.org", + "repository": "Optimization Online", + "url": "https://optimization-online.org/?p=22890", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://optimization-online.org/2018/03/6538/", + "detectedItemType": "preprint", + "items": [ + { + "itemType": "preprint", + "title": "Discretization-based algorithms for generalized semi-infinite and bilevel programs with coupling equality constraints", + "creators": [ + { + "firstName": "Hatim", + "lastName": "Djelassi", + "creatorType": "author" + }, + { + "firstName": "Alexander", + "lastName": "Mitsos", + "creatorType": "author" + }, + { + "firstName": "Moll", + "lastName": "Glass", + "creatorType": "author" + } + ], + "date": "2018-03-22", + "archiveID": "15107", + "language": "en-US", + "libraryCatalog": "optimization-online.org", + "repository": "Optimization Online", + "url": "https://optimization-online.org/?p=15107", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://optimization-online.org/tag/affine-recourse-adaptation/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/author/melvynsim/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/category/robust-optimization/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/2022/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/2022/07/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/2022/7/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/2022/07/1/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/2022/7/01/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/2022/7/1/", + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://optimization-online.org/2022/07/01/", + "detectedItemType": "multiple", + "items": "multiple" + } +] +/** END TEST CASES **/ diff --git a/Perlego.js b/Perlego.js new file mode 100644 index 00000000000..a29b2bb3ea4 --- /dev/null +++ b/Perlego.js @@ -0,0 +1,416 @@ +{ + "translatorID": "4fbb8dfd-459d-445e-bd2a-5ea89814b0c0", + "label": "Perlego", + "creator": "Brendan O'Connell", + "target": "^https?://(www\\.)?perlego\\.com", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-04-26 14:52:00" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Brendan O'Connell + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see <http://www.gnu.org/licenses/>. + + ***** END LICENSE BLOCK ***** +*/ + + +function detectWeb(doc, url) { + if (url.includes('/book/')) { + return 'book'; + } + else if (url.includes("/browse/") || url.includes("/search?") || url.includes("/publisher/") || url.includes("/reading-list/")) { + if (getSearchResults(doc, true)) { + return 'multiple'; + } + } + Z.monitorDOMChanges(doc.querySelector('div#root'), { childList: true, subtree: true }); + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + // limitation: this selector works well for /search?, /browse/, and /publisher/ pages, but doesn't work + // for /reading-list/, e.g. https://www.perlego.com/reading-list/86/introduction-to-social-movements?queryID=21da233727a255f6d709ad565bddc362 + var rows = doc.querySelectorAll('a[href*="/book/"]'); + for (let row of rows) { + var href = row.href; + // for non-logged in users, row.href sometimes contains /null/ so user sees a 404 error instead of the book + // remove this so we get to the correct URL + if (href.includes("null/")) { + href = href.replace("null/", ""); + } + let title = text(row, 'span'); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrape(doc, url = doc.location.href) { + let item = new Zotero.Item('book'); + const id = url.match(/\/book\/(\d+)/)[1]; + let apiUrl = "https://api.perlego.com/metadata/v2/metadata/books/" + id; + var apiJson = await requestJSON(apiUrl); + var metadata = apiJson.data.results[0]; + if (metadata.subtitle) { + item.title = metadata.title + ": " + metadata.subtitle; + } + else { + item.title = metadata.title; + } + item.shortTitle = metadata.title; + item.ISBN = ZU.cleanISBN(metadata.isbn13); + // multiple authors are entered in a single field in JSON, separated by comma or ampersand + let authors = metadata.author.split(/[,&]/); + for (let author of authors) { + item.creators.push(ZU.cleanAuthor(author, 'author')); + } + item.abstractNote = ZU.cleanTags(metadata.description); + let dateString = metadata.date; + let formattedDate = dateString.replace(/(\d{4})(\d{2})(\d{2})/, "$1-$2-$3"); + item.date = formattedDate; + item.place = metadata.publication_city; + item.edition = metadata.edition_number; + item.publisher = metadata.publisher_name; + item.language = metadata.language; + item.url = url; + if (metadata.subject[0].subject_name) { + item.tags.push(metadata.subject[0].subject_name); + } + if (metadata.topics[0].topic_name) { + item.tags.push(metadata.topics[0].topic_name); + } + item.complete(); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://www.perlego.com/book/781635/unshakeable-your-guide-to-financial-freedom-pdf", + "detectedItemType": "book", + "items": [ + { + "itemType": "book", + "title": "Unshakeable: Your Guide to Financial Freedom", + "creators": [ + { + "firstName": "Tony", + "lastName": "Robbins", + "creatorType": "author" + }, + { + "firstName": "Peter", + "lastName": "Mallouk", + "creatorType": "author" + } + ], + "date": "2017-02-28", + "ISBN": "9781471164941", + "abstractNote": "*THE NEW YORK TIMES BESTSELLER* Tony Robbins, arguably the most recognizable life and business strategist and guru, is back with a timely, unique follow-up to his smash New York Times bestseller Money: Master the Game.\n Market corrections are as constant as seasons are in nature. There have been 30 such corrections in the past 30 years, yet there's never been an action plan for how not only to survive, but thrive through each change in the stock market.\n Building upon the principles in Money: Master the Game, Robbins offers the reader specific steps they can implement to protect their investments while maximizing their wealth. It's a detailed guidedesigned for investors, articulated in the common-sense, practical manner that the millions of loyal Robbins fans and students have come to expect and rely upon.\n Few have navigated the turbulence of the stock market as adeptly and successfully as Tony Robbins. His proven, consistent success over decades makes him singularly qualified to help investors (both seasoned and first-timers alike) preserve and add to their investments. 'Tony's power is super-human' Oprah Winfrey\n 'He has a great gift. He has the gift to inspire' Bill Clinton\n 'Tony Robbins needs no introduction. He is committed to helping make life better for every investor' Carl Icahn\n 'The high priest of human potential. The world can't get enough of Anthony Robbins' The New York Times", + "language": "English", + "libraryCatalog": "Perlego", + "publisher": "Simon & Schuster UK", + "shortTitle": "Unshakeable", + "url": "https://www.perlego.com/book/781635/unshakeable-your-guide-to-financial-freedom-pdf", + "attachments": [], + "tags": [ + { + "tag": "Business" + }, + { + "tag": "Business General" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.perlego.com/book/2997236/operations-management-an-international-perspective-pdf", + "items": [ + { + "itemType": "book", + "title": "Operations Management: An International Perspective", + "creators": [ + { + "firstName": "David", + "lastName": "Barnes", + "creatorType": "author" + } + ], + "date": "2018-01-30", + "ISBN": "9781350305212", + "abstractNote": "This fascinating new core textbook, authored by a highly respected academic with over a decade of industry experience, takes a global and strategic approach to the important topic of operations management (OM). Integrating contemporary and traditional theories the text covers everything a student needs to understand the reality of operations in the modern world and combines the latest cutting-edge thinking with innovative learning features. Written in a concise and engaging style and based on up-to-date research in the field, the book provides a range of international case studies and examples that help students to apply theoretical knowledge to real-world practice. This is a must-have textbook for students studying operations management modules on undergraduate, postgraduate and MBA programmes. In addition, this is an ideal textbook to accompany modules on operations strategy, production management and services management.", + "edition": "1", + "language": "English", + "libraryCatalog": "Perlego", + "publisher": "Bloomsbury Academic", + "shortTitle": "Operations Management", + "url": "https://www.perlego.com/book/2997236/operations-management-an-international-perspective-pdf", + "attachments": [], + "tags": [ + { + "tag": "Business" + }, + { + "tag": "Operations" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.perlego.com/book/2306297/mathematics-n1-students-book-tvet-first-pdf", + "items": [ + { + "itemType": "book", + "title": "Mathematics N1 Student's Book: TVET FIRST", + "creators": [ + { + "firstName": "MJJ van", + "lastName": "Rensburg", + "creatorType": "author" + } + ], + "date": "2014-01-01", + "ISBN": "9781430804017", + "abstractNote": "A top-rated series of textbooks designed to help students reach their highest potential. Easy to follow with logical sequencing and a step-by-step approach to problem-solving. Comprehensive module summaries, detailed worked examples and plenty of activities to prepare students for exams. Lots of typical exam-type questions so students understand what is expected of them. Simple defi niti ons for new terms to remove language barriers.", + "language": "English", + "libraryCatalog": "Perlego", + "publisher": "Troupant", + "shortTitle": "Mathematics N1 Student's Book", + "url": "https://www.perlego.com/book/2306297/mathematics-n1-students-book-tvet-first-pdf", + "attachments": [], + "tags": [ + { + "tag": "Engineering General" + }, + { + "tag": "Technology & Engineering" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.perlego.com/book/862303/designing-experiences-pdf", + "items": [ + { + "itemType": "book", + "title": "Designing Experiences", + "creators": [ + { + "firstName": "J. Robert", + "lastName": "Rossman", + "creatorType": "author" + }, + { + "firstName": "Mathew D.", + "lastName": "Duerden", + "creatorType": "author" + } + ], + "date": "2019-07-23", + "ISBN": "9780231549516", + "abstractNote": "In an increasingly experience-driven economy, companies that deliver great experiences thrive, and those that do not die. Yet many organizations face difficulties implementing a vision of delivering experiences beyond the provision of goods and services while students and aspiring professionals struggle to piece together the principles of experience design from disparate, often disconnected disciplines and approaches.In this book, J. Robert Rossman and Mathew D. Duerden present a comprehensive and accessible introduction to experience design. They synthesize the fundamental theories and methods from multiple disciplines and lay out a process for designing experiences from start to finish. Rossman and Duerden challenge us to reflect on what makes a great experience from the user's perspective, drawing attention to both the macro and micro levels. They present interdisciplinary research underlying key concepts such as memory, intentionality, and dramatic structure in a down-to-earth style. Designing Experiences features detailed instructions and numerous real-world examples that clarify theoretical principles, making it useful for students and professionals. An invaluable overview of a growing field, the book provides readers with the tools they need to design innovative and indelible experiences.", + "language": "English", + "libraryCatalog": "Perlego", + "publisher": "Columbia University Press", + "url": "https://www.perlego.com/book/862303/designing-experiences-pdf", + "attachments": [], + "tags": [ + { + "tag": "Business" + }, + { + "tag": "Management" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.perlego.com/book/1809187/war-and-peace-pdf", + "items": [ + { + "itemType": "book", + "title": "War and Peace", + "creators": [ + { + "firstName": "Leo", + "lastName": "Tolstoy", + "creatorType": "author" + } + ], + "date": "2020-09-08", + "ISBN": "9781528791342", + "abstractNote": "One of the most famous examples of classic world literature, Tolstoy's \"War and Peace\" is an epic chronicle of France's invasion of Russia and the aftermath of the Napoleonic era on Russian society as experienced by five families belonging to the aristocracy. Originally released in serial form in \"The Russian Messenger\" between 1865 and 1867, \"War and Peace\" is considered to be among Tolstoy's greatest literary works and constitutes an absolute must-read for all literature lovers. Count Lev Nikolayevich Tolstoy (1828–1910), also known in English as Leo Tolstoy, was a Russian writer. Generally considered to be one among the greatest novel writers of all time, he was nominated for the Nobel Prize in literature each year between 1902 and 1906; as well as the Nobel Peace Prize in 1901, 1902, and 1910. Other notable works by this author include: \"Anna Karenina\" (1877), \"The Cossacks\" (1863), and \"Resurrection\" (1899). Read & Co. Classics is proudly republishing this classic novel now in a new edition complete with a specially-commissioned new biography of the author.", + "language": "English", + "libraryCatalog": "Perlego", + "publisher": "Read & Co. Classics", + "url": "https://www.perlego.com/book/1809187/war-and-peace-pdf", + "attachments": [], + "tags": [ + { + "tag": "Classics" + }, + { + "tag": "Literature" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.perlego.com/browse/literature?language=All&publicationDate=&publisher=&author=&format=&tab=books&page=1", + "defer": true, + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.perlego.com/book/2179957/chor-und-theorie-zeitgenssische-theatertexte-von-heiner-mller-bis-ren-pollesch-pdf", + "detectedItemType": "book", + "items": [ + { + "itemType": "book", + "title": "Chor und Theorie: Zeitgenössische Theatertexte von Heiner Müller bis René Pollesch", + "creators": [ + { + "firstName": "Maria", + "lastName": "Kuberg", + "creatorType": "author" + } + ], + "date": "2021-02-22", + "ISBN": "9783835397361", + "abstractNote": "Nicht nur zwischen Heimat und Fremde steht der Chor im Theater, sondern auch zwischen selbstständigem Handeln und Abhängigkeit, zwischen Menschen und Göttern und schließlich zwischen der dargestellten Welt und der Realität der Rezipienten. Am Chor entzündet sich die Frage, wie die Gemeinschaft auftritt.Maria Kuberg untersucht in detaillierten Lektüren chorischer Theaterstücke von Heiner Müller, Botho Strauß, Elfriede Jelinek, Tankred Dorst, Ewald Palmetshofer, Rainald Goetz, Gert Jonke und René Pollesch, wie der Chor in zeitgenössischen deutschsprachigen Theatertexten zur Sprache kommt. Welche Formen nimmt die Chor-Gemeinschaft dabei im Text an? Und wie korrespondieren diese mit der dramatischen Gattung, die doch grundsätzlich die Handlungen Einzelner vorführt? Das sind die leitenden Fragen dieser erhellenden Erkundung des Theaters der Gegenwart, die es als Reflexion über Gemeinschaft profiliert.Dabei operiert die Untersuchung auf drei historischen Ebenen: Chorische Theatertexte aus dem späten 20. und frühen 21. Jahrhundert werden mit einer theatralen Tradition konfrontiert, die bis in die griechische Antike zurückreicht. Zwischen den antiken und den aktuellen Texten vermittelt die philosophisch-ästhetische Auseinandersetzung mit dem Chor, wie sie im 19. Jahrhundert Schiller, A. W. und F. Schlegel, Hegel und Nietzsche führen. Im Zusammenspiel dieser drei Ebenen wird so eine Theorie des Chorischen entwickelt, die Gattungsaspekte und Gemeinschaftstheorien gleichermaßen berücksichtigt und die das ästhetische wie auch das politische Potenzial der untersuchten Texte erschließt.", + "language": "German", + "libraryCatalog": "Perlego", + "publisher": "Konstanz University Press", + "shortTitle": "Chor und Theorie", + "url": "https://www.perlego.com/book/2179957/chor-und-theorie-zeitgenssische-theatertexte-von-heiner-mller-bis-ren-pollesch-pdf", + "attachments": [], + "tags": [ + { + "tag": "Literary Criticism for Comparative Literature" + }, + { + "tag": "Literature" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.perlego.com/browse/social-sciences/social-science-research-methodology?language=All&publicationDate=&publisher=&author=&format=&page=1", + "defer": true, + "detectedItemType": "multiple", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.perlego.com/book/1/sheri-khan-tarakai-and-early-village-life-in-the-borderlands-of-northwest-pakistan-bannu-archaeological-project-surveys-and-excavations-19852001-pdf", + "detectedItemType": "book", + "items": [ + { + "itemType": "book", + "title": "Sheri Khan Tarakai and Early Village Life in the Borderlands of North-West Pakistan: Bannu Archaeological Project Surveys and Excavations 1985-2001", + "creators": [ + { + "firstName": "Cameron A.", + "lastName": "Petrie", + "creatorType": "author" + }, + { + "firstName": "Farid", + "lastName": "Khan", + "creatorType": "author" + }, + { + "firstName": "J. R.", + "lastName": "Knox", + "creatorType": "author" + } + ], + "date": "2010-05-10", + "ISBN": "9781842177358", + "abstractNote": "Between 1985 and 2001, the collaborative research initiative known as the Bannu Archaeological Project conducted archaeological explorations and excavations in the Bannu region, in what was then the North West Frontier Province (NWFP) of Pakistan, now Khyber-Pakhtunkhwa. The Project involves scholars from the Pakistan Heritage Society, the British Museum, the Institute of Archaeology (UCL), Bryn Mawr College and the University of Cambridge. This is the first in a series of volumes that present the final reports of the exploration and excavations carried out by the Bannu Archaeological Project. It marks the first attempt to contextualise the earliest village settlements in northwest Pakistan, along with those situated in other parts of the borderlands zone at the western margins of South Asia. An extensive range of archaeological data from the Bannu Archaeological Project excavations at Sheri Khan Tarakai, including stratigraphic, architectural, ceramic, lithic, small find and bioarchaeological elements, are presented, along with the results of surveys and excavations at several other sites in the Bannu Basin and the adjacent Gomal Plain. The work establishes the nature of the relationships between these sites and other early villages elsewhere in South, central and greater West Asia.", + "language": "English", + "libraryCatalog": "Perlego", + "place": "Havertown", + "publisher": "Oxbow Books", + "shortTitle": "Sheri Khan Tarakai and Early Village Life in the Borderlands of North-West Pakistan", + "url": "https://www.perlego.com/book/1/sheri-khan-tarakai-and-early-village-life-in-the-borderlands-of-northwest-pakistan-bannu-archaeological-project-surveys-and-excavations-19852001-pdf", + "attachments": [], + "tags": [ + { + "tag": "Asian History" + }, + { + "tag": "History" + } + ], + "notes": [], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/PhilPapers.js b/PhilPapers.js index f9406dbaa00..fed4465ed9d 100644 --- a/PhilPapers.js +++ b/PhilPapers.js @@ -9,93 +9,109 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-08-13 23:13:58" + "lastUpdated": "2023-09-16 03:30:01" } /* ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2012 Sebastian Karcher + + Copyright © 2023 Sebastian Karcher + This file is part of Zotero. - + Zotero is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. - + Zotero is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. - + You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see <http://www.gnu.org/licenses/>. - + along with Zotero. If not, see <http://www.gnu.org/licenses/>. + ***** END LICENSE BLOCK ***** */ + function detectWeb(doc, url) { - if (/\/s|pub\//.test(url)) return "multiple"; - if (url.includes("/browse/") && ZU.xpathText(doc, '//ol[@class="entryList"]/li/@id') !== null) return "multiple"; - if (url.includes("/rec/")) return "journalArticle"; + if (url.includes('/rec/')) { + return 'journalArticle'; + } + else if (getSearchResults(doc, true)) { + return 'multiple'; + } return false; } - -function doWeb(doc, url) { - let isPhilArchive = /^https?:\/\/philarchive\.org\//.test(url); +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('.entryList .citation>a'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} +function idFromUrl(url) { + return url.match(/\/rec\/([A-Z-\d]+)/)[1]; +} +async function doWeb(doc, url) { + let isPhilArchive = /^https?:\/\/philarchive\.org\//.test(url); var ids = []; - if (detectWeb(doc, url) == "multiple") { - var items = {}; - var titles = ZU.xpath(doc, '//li/span[@class="citation"]//span[contains (@class, "articleTitle")]'); - var identifiers = ZU.xpath(doc, '//ol[@class="entryList"]/li/@id'); - for (var i in titles) { - items[identifiers[i].textContent] = titles[i].textContent; + + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + let id = idFromUrl(url); + ids.push(id); } - Zotero.selectItems(items, function (items) { - if (!items) { - return; - } - for (var i in items) { - ids.push(i.replace(/^e/, "")); - } - scrape(ids, isPhilArchive); - }); + await scrape(ids, isPhilArchive); } else { - var identifier = url.match(/(\/rec\/)([A-Z-\d]+)/)[2]; + let identifier = idFromUrl(url); // Z.debug(identifier) - scrape([identifier], isPhilArchive); + await scrape([identifier], isPhilArchive); } } -function scrape(identifiers, isPhilArchive) { + +async function scrape(identifiers, isPhilArchive) { + let baseUrl = isPhilArchive ? "https://philarchive.org" : "https://philpapers.org"; for (let id of identifiers) { - let bibtexURL = "/export.html?__format=bib&eId=" + id + "&formatName=BibTeX"; - Zotero.Utilities.HTTP.doGet(bibtexURL, function (text) { - // remove line breaks, then match match the bibtex. - var bibtex = text.replace(/\n/g, "").match(/<pre class='export'>.+<\/pre>/)[0]; - var url = "/rec/" + id; - var translator = Zotero.loadTranslator("import"); - translator.setTranslator("9cb70025-a888-4a29-a210-93ec52da40d4"); - translator.setString(bibtex); - translator.setHandler("itemDone", function (obj, item) { - if (isPhilArchive) { - item.libraryCatalog = 'PhilArchive'; - item.url = `https://philarchive.org/rec/${id}`; // full-text - item.attachments.push({ - title: 'Full Text PDF', - mimeType: 'application/pdf', - url: `/archive/${id}` - }); - } - - item.attachments.push({ url, title: "Snapshot", mimeType: "text/html" }); - item.complete(); - }); - translator.translate(); + let bibUrl = `${baseUrl}/item.pl?eId=${id}&format=bib`; + let bibText = await requestText(bibUrl); + let url = "/rec/" + id; + let translator = Zotero.loadTranslator("import"); + translator.setTranslator('9cb70025-a888-4a29-a210-93ec52da40d4'); + translator.setString(bibText); + translator.setHandler('itemDone', (_obj, item) => { + if (isPhilArchive) { + item.libraryCatalog = 'PhilArchive'; + item.url = `https://philarchive.org/rec/${id}`; // full-text + item.attachments.push({ + title: 'Full Text PDF', + mimeType: 'application/pdf', + url: `/archive/${id}` + }); + } + else { + item.attachments.push({ url, + title: 'Snapshot', + mimeType: 'text/html' }); + } + item.complete(); }); + await translator.translate(); } } @@ -107,7 +123,7 @@ var testCases = [ "items": [ { "itemType": "journalArticle", - "title": "Observation, Character, and A Purely First-Person Point of View", + "title": "Observation, Character, and a Purely First-Person Point of View", "creators": [ { "firstName": "Josep E.", @@ -156,7 +172,7 @@ var testCases = [ "items": [ { "itemType": "journalArticle", - "title": "Norm-Based Governance for a New Era: Collective Action in the Face of Hyper-Politicization", + "title": "Norm-Based Governance for a New Era: Lessons From Climate Change and Covid-19", "creators": [ { "firstName": "Leigh", @@ -174,19 +190,18 @@ var testCases = [ "creatorType": "author" } ], - "itemID": "RaymondForthcoming-RAYNGF", + "date": "2021", + "itemID": "Raymond2021-RAYNGF", "libraryCatalog": "PhilArchive", + "pages": "1–14", "publicationTitle": "Perspectives on Politics", "shortTitle": "Norm-Based Governance for a New Era", "url": "https://philarchive.org/rec/RAYNGF", + "volume": "1", "attachments": [ { "title": "Full Text PDF", "mimeType": "application/pdf" - }, - { - "title": "Snapshot", - "mimeType": "text/html" } ], "tags": [], @@ -200,7 +215,7 @@ var testCases = [ "url": "https://philarchive.org/rec/LANTEO-39", "items": [ { - "itemType": "manuscript", + "itemType": "journalArticle", "title": "The Ethics of Partiality", "creators": [ { @@ -209,17 +224,19 @@ var testCases = [ "creatorType": "author" } ], - "itemID": "LangeManuscript-LANTEO-39", + "date": "2022", + "DOI": "10.1111/phc3.12860", + "issue": "8", + "itemID": "Lange2022-LANTEO-39", "libraryCatalog": "PhilArchive", + "pages": "1–15", + "publicationTitle": "Philosophy Compass", "url": "https://philarchive.org/rec/LANTEO-39", + "volume": "1", "attachments": [ { "title": "Full Text PDF", "mimeType": "application/pdf" - }, - { - "title": "Snapshot", - "mimeType": "text/html" } ], "tags": [], diff --git a/Preprints.org.js b/Preprints.org.js index a2432110e22..f29d0a26d96 100644 --- a/Preprints.org.js +++ b/Preprints.org.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-08-31 14:35:47" + "lastUpdated": "2023-08-23 07:21:42" } /* @@ -73,11 +73,9 @@ function getSearchResults(doc, checkOnly) { async function doWeb(doc, url) { if (detectWeb(doc, url) == 'multiple') { let items = await Zotero.selectItems(getSearchResults(doc, false)); - if (items) { - await Promise.all( - Object.keys(items) - .map(url => requestDocument(url).then(scrape)) - ); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } } else { @@ -87,10 +85,13 @@ async function doWeb(doc, url) { async function scrape(doc, url) { if (doc.querySelector('.peer-reviewed-box')) { - let DOI = doc.querySelector('.peer-reviewed-box .journal-ref').lastChild.nodeValue; - if (DOI) DOI = ZU.cleanDOI(DOI); - if (DOI) { - await scrapeDOI(DOI); + let doiNode = doc.querySelector('.peer-reviewed-box .journal-ref'); + if (doiNode) { + let DOI = doiNode.lastChild.nodeValue; + if (DOI) DOI = ZU.cleanDOI(DOI); + if (DOI) { + await scrapeDOI(DOI); + } } else { await scrapePublisher(attr(doc, '.peer-reviewed-box a', 'href')); diff --git a/ProQuest.js b/ProQuest.js index 84f4979cb5f..4672958fac7 100644 --- a/ProQuest.js +++ b/ProQuest.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-12-15 15:34:27" + "lastUpdated": "2023-08-24 15:19:14" } /* @@ -498,10 +498,11 @@ function scrape(doc, url, type) { item.tags = altKeywords.join(',').split(/\s*(?:,|;)\s*/); } - if (doc.getElementById('downloadPDFLink')) { + let pdfLink = doc.querySelector('[id^="downloadPDFLink"]'); + if (pdfLink) { item.attachments.push({ title: 'Full Text PDF', - url: doc.getElementById('downloadPDFLink').href, + url: pdfLink.href, mimeType: 'application/pdf', proxy: false }); diff --git a/RIS.js b/RIS.js index 9fd7761af8f..a7f5a65fb65 100644 --- a/RIS.js +++ b/RIS.js @@ -17,7 +17,7 @@ }, "inRepository": true, "translatorType": 3, - "lastUpdated": "2023-04-20 19:03:44" + "lastUpdated": "2023-07-28 09:46:04" } /* @@ -1252,15 +1252,15 @@ var CitaviCleaner = new function () { if (!entry.tags.H1) { // Only have a call number (maybe multiple, so take the first) - let at = entry.tags.indexOf(entry.tags.H2[0]); - TagCleaner.changeTag(entry, at, 'CN'); + let at = entry.indexOf(entry.tags.H2[0]); + TagCleaner.changeTag(entry, at, ['CN']); return; } if (!entry.tags.H1) { // Only have a library - let at = entry.tags.indexOf(entry.tags.H1[0]); - TagCleaner.changeTag(entry, at, 'DP'); + let at = entry.indexOf(entry.tags.H1[0]); + TagCleaner.changeTag(entry, at, ['DP']); } // We have pairs, so find the first set and change it @@ -7240,6 +7240,26 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "import", + "input": "TY - GEN\nT1 - Citavi Item\nKW - Tag\nH2 - Call Number\nM4 - Citavi\nER -", + "items": [ + { + "itemType": "journalArticle", + "title": "Citavi Item", + "creators": [], + "callNumber": "Call Number", + "attachments": [], + "tags": [ + { + "tag": "Tag" + } + ], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/ReferBibIX.js b/ReferBibIX.js index 66b80dc01ca..bfcc9bf5ac5 100644 --- a/ReferBibIX.js +++ b/ReferBibIX.js @@ -11,7 +11,7 @@ }, "inRepository": true, "translatorType": 3, - "lastUpdated": "2021-10-23 20:56:30" + "lastUpdated": "2023-10-27 09:03:42" } function detectImport() { @@ -104,6 +104,7 @@ var typeMap = { // TODO: BILL, CASE, COMP, CONF, DATA, HEAR, MUSIC, PAT, SOUND, STAT var inputTypeMap = { "Ancient Text": "book", + Audio: "audioRecording", "Audiovisual Material": "videoRecording", Generic: "book", "Chart or Table": "artwork", @@ -437,6 +438,26 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "import", + "input": "%0 Serial\n%0 Audio\n%I Studio Omega, Verein für Christliche Radioarbeit\n%D 2021\n%C Wien\n%G German\n%T Diesseits von Eden: der Podcast der katholischen Fakultäten Österreichs & Südtirols\n%U https://diesseits.theopodcast.at/home", + "items": [ + { + "itemType": "audioRecording", + "title": "Diesseits von Eden: der Podcast der katholischen Fakultäten Österreichs & Südtirols", + "creators": [], + "date": "2021", + "label": "Studio Omega, Verein für Christliche Radioarbeit", + "language": "German", + "place": "Wien", + "url": "https://diesseits.theopodcast.at/home", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/Scopus.js b/Scopus.js index 5758d43374c..6939bfe0a19 100644 --- a/Scopus.js +++ b/Scopus.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-11-09 13:19:39" + "lastUpdated": "2023-08-25 02:42:41" } /* @@ -48,7 +48,7 @@ function getEID(url) { function getSearchResults(doc, checkOnly) { var items = {}; var found = false; - var rows = doc.querySelectorAll('tr[id *= resultDataRow] td a[title = "Show document details"], tr[class *= "resultsRow"] h4 a[title = "Show document details"]'); + var rows = doc.querySelectorAll('tr[id *= resultDataRow] td a[title = "Show document details"], tr[class *= "resultsRow"] h4 a[title = "Show document details"], div.table-title h4 a'); for (var i = 0; i < rows.length; i++) { var href = rows[i].href; var title = ZU.trimInternal(rows[i].textContent); diff --git a/Taylor and Francis+NEJM.js b/Taylor and Francis+NEJM.js index 820957f3e19..24669eede8b 100644 --- a/Taylor and Francis+NEJM.js +++ b/Taylor and Francis+NEJM.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-07-19 15:27:10" + "lastUpdated": "2023-10-27 16:03:23" } /* @@ -38,7 +38,7 @@ function detectWeb(doc, url) { - if (url.match(/\/doi\/(abs|full|figure)\/10\./)) { + if (url.match(/\/doi(\/(abs|full|figure))?\/10\./)) { return "journalArticle"; } else if ((url.includes('/action/doSearch?') || url.includes('/toc/')) && getSearchResults(doc, true)) { @@ -88,7 +88,7 @@ function doWeb(doc, url) { function scrape(doc, url) { - var match = url.match(/\/doi\/(?:abs|full|figure)\/(10\.[^?#]+)/); + var match = url.match(/\/doi(?:\/(?:abs|full|figure))?\/(10\.[^?#]+)/); var doi = match[1]; var baseUrl = url.match(/https?:\/\/[^/]+/)[0]; @@ -585,6 +585,43 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://www.nejm.org/doi/10.1056/NEJMcibr2307735", + "items": [ + { + "itemType": "journalArticle", + "title": "A Holy Grail — The Prediction of Protein Structure", + "creators": [ + { + "lastName": "Altman", + "firstName": "Russ B.", + "creatorType": "author" + } + ], + "date": "2023-10-12", + "DOI": "10.1056/NEJMcibr2307735", + "ISSN": "0028-4793", + "extra": "PMID: 37732608", + "issue": "15", + "itemID": "doi:10.1056/NEJMcibr2307735", + "libraryCatalog": "Taylor and Francis+NEJM", + "pages": "1431-1434", + "publicationTitle": "New England Journal of Medicine", + "url": "https://doi.org/10.1056/NEJMcibr2307735", + "volume": "389", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/The Atlantic.js b/The Atlantic.js index 1b1dc3b49b8..1e85f5b3af7 100644 --- a/The Atlantic.js +++ b/The Atlantic.js @@ -2,23 +2,20 @@ "translatorID": "575ba37f-c871-4ee8-8bdb-3e7f954e4e6a", "label": "The Atlantic", "creator": "Sebastian Karcher", - "target": "^https?://www\\.theatlantic\\.com", - "minVersion": "2.1.9", + "target": "^https://www\\.theatlantic\\.com/.+", + "minVersion": "5.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, - "browserSupport": "gcsib", - "lastUpdated": "2016-12-27 20:33:39" + "browserSupport": "gcsibv", + "lastUpdated": "2023-09-16 01:54:13" } -/* FW LINE 59:b820c6d */ function flatten(t){var e=new Array;for(var i in t){var r=t[i];r instanceof Array?e=e.concat(flatten(r)):e.push(r)}return e}var FW={_scrapers:new Array};FW._Base=function(){this.callHook=function(t,e,i,r){if("object"==typeof this.hooks){var n=this.hooks[t];"function"==typeof n&&n(e,i,r)}},this.evaluateThing=function(t,e,i){var r=typeof t;if("object"===r){if(t instanceof Array){var n=this.evaluateThing,a=t.map(function(t){return n(t,e,i)});return flatten(a)}return t.evaluate(e,i)}return"function"===r?t(e,i):t},this.makeItems=function(t,e,i,r,n){n()}},FW.Scraper=function(t){FW._scrapers.push(new FW._Scraper(t))},FW._Scraper=function(t){for(x in t)this[x]=t[x];this._singleFieldNames=["abstractNote","applicationNumber","archive","archiveLocation","artworkMedium","artworkSize","assignee","audioFileType","audioRecordingType","billNumber","blogTitle","bookTitle","callNumber","caseName","code","codeNumber","codePages","codeVolume","committee","company","conferenceName","country","court","date","dateDecided","dateEnacted","dictionaryTitle","distributor","docketNumber","documentNumber","DOI","edition","encyclopediaTitle","episodeNumber","extra","filingDate","firstPage","forumTitle","genre","history","institution","interviewMedium","ISBN","ISSN","issue","issueDate","issuingAuthority","journalAbbreviation","label","language","legalStatus","legislativeBody","letterType","libraryCatalog","manuscriptType","mapType","medium","meetingName","nameOfAct","network","number","numberOfVolumes","numPages","pages","patentNumber","place","postType","presentationType","priorityNumbers","proceedingsTitle","programTitle","programmingLanguage","publicLawNumber","publicationTitle","publisher","references","reportNumber","reportType","reporter","reporterVolume","rights","runningTime","scale","section","series","seriesNumber","seriesText","seriesTitle","session","shortTitle","studio","subject","system","thesisType","title","type","university","url","version","videoRecordingType","volume","websiteTitle","websiteType"],this._makeAttachments=function(t,e,i,r){if(i instanceof Array)i.forEach(function(i){this._makeAttachments(t,e,i,r)},this);else if("object"==typeof i){var n=i.urls||i.url,a=i.types||i.type,s=i.titles||i.title,o=i.snapshots||i.snapshot,u=this.evaluateThing(n,t,e),l=this.evaluateThing(s,t,e),c=this.evaluateThing(a,t,e),h=this.evaluateThing(o,t,e);u instanceof Array||(u=[u]);for(var f in u){var p,m,v,d=u[f];p=c instanceof Array?c[f]:c,m=l instanceof Array?l[f]:l,v=h instanceof Array?h[f]:h,r.attachments.push({url:d,title:m,mimeType:p,snapshot:v})}}},this.makeItems=function(t,e,i,r,n){var a=new Zotero.Item(this.itemType);a.url=e;for(var s in this._singleFieldNames){var o=this._singleFieldNames[s];if(this[o]){var u=this.evaluateThing(this[o],t,e);u instanceof Array?a[o]=u[0]:a[o]=u}}var l=["creators","tags"];for(var c in l){var h=l[c],f=this.evaluateThing(this[h],t,e);if(f)for(var p in f)a[h].push(f[p])}this._makeAttachments(t,e,this.attachments,a),r(a,this,t,e),n()}},FW._Scraper.prototype=new FW._Base,FW.MultiScraper=function(t){FW._scrapers.push(new FW._MultiScraper(t))},FW._MultiScraper=function(t){for(x in t)this[x]=t[x];this._mkSelectItems=function(t,e){var i=new Object;for(var r in t)i[e[r]]=t[r];return i},this._selectItems=function(t,e,i){var r=new Array;Zotero.selectItems(this._mkSelectItems(t,e),function(t){for(var e in t)r.push(e);i(r)})},this._mkAttachments=function(t,e,i){var r=this.evaluateThing(this.attachments,t,e),n=new Object;if(r)for(var a in i)n[i[a]]=r[a];return n},this._makeChoices=function(t,e,i,r,n){if(t instanceof Array)t.forEach(function(t){this._makeTitlesUrls(t,e,i,r,n)},this);else if("object"==typeof t){var a=t.urls||t.url,s=t.titles||t.title,o=this.evaluateThing(a,e,i),u=this.evaluateThing(s,e,i),l=u instanceof Array;o instanceof Array||(o=[o]);for(var c in o){var h,f=o[c];h=l?u[c]:u,n.push(f),r.push(h)}}},this.makeItems=function(t,e,i,r,n){if(this.beforeFilter){var a=this.beforeFilter(t,e);if(a!=e)return void this.makeItems(t,a,i,r,n)}var s=[],o=[];this._makeChoices(this.choices,t,e,s,o);var u=this._mkAttachments(t,e,o),l=this.itemTrans;this._selectItems(s,o,function(t){if(t){var e=function(t){var e=t.documentURI,i=l;void 0===i&&(i=FW.getScraper(t,e)),void 0===i||i.makeItems(t,e,u[e],r,function(){})};Zotero.Utilities.processDocuments(t,e,n)}else n()})}},FW._MultiScraper.prototype=new FW._Base,FW.WebDelegateTranslator=function(t){return new FW._WebDelegateTranslator(t)},FW._WebDelegateTranslator=function(t){for(x in t)this[x]=t[x];this.makeItems=function(t,e,i,r,n){var a=this,s=Zotero.loadTranslator("web");s.setHandler("itemDone",function(i,n){r(n,a,t,e)}),s.setDocument(t),this.translatorId?(s.setTranslator(this.translatorId),s.translate()):(s.setHandler("translators",function(t,e){e.length&&(s.setTranslator(e[0]),s.translate())}),s.getTranslators()),n()}},FW._WebDelegateTranslator.prototype=new FW._Base,FW._StringMagic=function(){this._filters=new Array,this.addFilter=function(t){return this._filters.push(t),this},this.split=function(t){return this.addFilter(function(e){return e.split(t).filter(function(t){return""!=t})})},this.replace=function(t,e,i){return this.addFilter(function(r){return r.match(t)?r.replace(t,e,i):r})},this.prepend=function(t){return this.replace(/^/,t)},this.append=function(t){return this.replace(/$/,t)},this.remove=function(t,e){return this.replace(t,"",e)},this.trim=function(){return this.addFilter(function(t){return Zotero.Utilities.trim(t)})},this.trimInternal=function(){return this.addFilter(function(t){return Zotero.Utilities.trimInternal(t)})},this.match=function(t,e){return e||(e=0),this.addFilter(function(i){var r=i.match(t);return void 0===r||null===r?void 0:r[e]})},this.cleanAuthor=function(t,e){return this.addFilter(function(i){return Zotero.Utilities.cleanAuthor(i,t,e)})},this.key=function(t){return this.addFilter(function(e){return e[t]})},this.capitalizeTitle=function(){return this.addFilter(function(t){return Zotero.Utilities.capitalizeTitle(t)})},this.unescapeHTML=function(){return this.addFilter(function(t){return Zotero.Utilities.unescapeHTML(t)})},this.unescape=function(){return this.addFilter(function(t){return unescape(t)})},this._applyFilters=function(t,e){for(i in this._filters){t=flatten(t),t=t.filter(function(t){return void 0!==t&&null!==t});for(var r=0;r<t.length;r++)try{if(void 0===t[r]||null===t[r])continue;t[r]=this._filters[i](t[r],e)}catch(n){t[r]=void 0,Zotero.debug("Caught exception "+n+"on filter: "+this._filters[i])}t=t.filter(function(t){return void 0!==t&&null!==t})}return flatten(t)}},FW.PageText=function(){return new FW._PageText},FW._PageText=function(){this._filters=new Array,this.evaluate=function(t){var e=[t.documentElement.innerHTML];return e=this._applyFilters(e,t),0==e.length?!1:e}},FW._PageText.prototype=new FW._StringMagic,FW.Url=function(){return new FW._Url},FW._Url=function(){this._filters=new Array,this.evaluate=function(t,e){var i=[e];return i=this._applyFilters(i,t),0==i.length?!1:i}},FW._Url.prototype=new FW._StringMagic,FW.Xpath=function(t){return new FW._Xpath(t)},FW._Xpath=function(t){this._xpath=t,this._filters=new Array,this.text=function(){var t=function(t){return"object"==typeof t&&t.textContent?t.textContent:t};return this.addFilter(t),this},this.sub=function(t){var e=function(e,i){var r=i.evaluate(t,e,null,XPathResult.ANY_TYPE,null);return r?r.iterateNext():void 0};return this.addFilter(e),this},this.evaluate=function(t){var e=t.evaluate(this._xpath,t,null,XPathResult.ANY_TYPE,null),i=e.resultType,r=new Array;if(i==XPathResult.STRING_TYPE)r.push(e.stringValue);else if(i==XPathResult.BOOLEAN_TYPE)r.push(e.booleanValue);else if(i==XPathResult.NUMBER_TYPE)r.push(e.numberValue);else if(i==XPathResult.ORDERED_NODE_ITERATOR_TYPE||i==XPathResult.UNORDERED_NODE_ITERATOR_TYPE)for(var n;n=e.iterateNext();)r.push(n);return r=this._applyFilters(r,t),0==r.length?!1:r}},FW._Xpath.prototype=new FW._StringMagic,FW.detectWeb=function(t,e){for(var i in FW._scrapers){var r=FW._scrapers[i],n=r.evaluateThing(r.itemType,t,e),a=r.evaluateThing(r.detect,t,e);if(a.length>0&&a[0])return n}},FW.getScraper=function(t,e){var i=FW.detectWeb(t,e);return FW._scrapers.filter(function(r){return r.evaluateThing(r.itemType,t,e)==i&&r.evaluateThing(r.detect,t,e)})[0]},FW.doWeb=function(t,e){var i=FW.getScraper(t,e);i.makeItems(t,e,[],function(t,e,i,r){e.callHook("scraperDone",t,i,r),t.title||(t.title=""),t.complete()},function(){Zotero.done()}),Zotero.wait()}; - /* ***** BEGIN LICENSE BLOCK ***** - The Atlantic Translator - Copyright © 2011 Sebastian Karcher + Copyright © 2011 Sebastian Karcher and contributors This file is part of Zotero. @@ -38,61 +35,210 @@ ***** END LICENSE BLOCK ***** */ -function detectWeb(doc, url) { return FW.detectWeb(doc, url); } -function doWeb(doc, url) { return FW.doWeb(doc, url); } - -/** Magazine */ -FW.Scraper({ -itemType : 'magazineArticle', -detect : FW.Xpath('//article[@id="article"]//h1[@class="hed"]'), -title : FW.Xpath('//article[@id="article"]//h1[@class="hed"]').text().trim(), -attachments : [{ url: FW.Url(), - title: "The Atlantic Snapshot", - type: "text/html" }], -creators : FW.Xpath('//div[@class="article-cover-extra"]//ul[@class="metadata"]/li[@class="byline"]').text().cleanAuthor("author"), -date : FW.Xpath('//div[@class="article-cover-extra"]//ul[@class="metadata"]/li[@class="date"]').text(), -abstractNote : FW.Xpath('//article//p[@class="dek"]').text().trim(), -publicationTitle : "The Atlantic", -ISSN : "1072-7825" -}); - - -/** Magazine Features */ -FW.Scraper({ -itemType : 'magazineArticle', -detect : FW.Xpath('//article[@class="article"]//h1[@class="hed"]'), -title : FW.Xpath('//article[@class="article"]//h1[@class="hed"]').text().trim(), -attachments : [{ url: FW.Url(), - title: "The Atlantic Snapshot", - type: "text/html" }], -creators : FW.Xpath('//header//address[@class="byline"]/a').text().cleanAuthor("author"), -date : FW.Xpath('//header/time[@class="date"]').text(), -abstractNote : FW.Xpath('//article//p[@class="dek"]').text().trim(), -publicationTitle : "The Atlantic", -ISSN : "1072-7825" -}); - - /** Search Results */ -FW.MultiScraper({ -itemType : 'multiple', -detect : FW.Url().match(/search\/\?q=/), -choices : { - titles : FW.Xpath('//a[@class="gs-title"]').text().trim(), - urls : FW.Xpath('//a[@class="gs-title"]').key("href").text() +// path at /[type]/archive/yyyy/mm/... +const SINGLE_ITEM_URL_RE = /^https:\/\/[^/]+\/(.+)\/archive\/\d{4}\/\d{2}\/.+/; + +function detectWeb(doc, url) { + let singleItemMatch = url.match(SINGLE_ITEM_URL_RE); + + if (!singleItemMatch) { + return getSearchResults(doc, true) && "multiple"; + } + + switch (singleItemMatch[1]) { + // articles from print issues + case "magazine": + return "magazineArticle"; + case "podcasts": + return "podcast"; + default: + // see, e.g. https://www.theatlantic.com/category/fiction/ + // and look at the class list of the li elements; the + // non-magazineArticle items have "blog-article" as one of its + // classes + return "blogPost"; // TODO: consider this + } +} + +function getSearchResults(doc, checkOnly) { + let items = {}; + let found = false; + // "li.article" selector: see https://www.theatlantic.com/category/fiction/ + let rows = doc.querySelectorAll("li.article > a, a[data-action~='title']"); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!title || !SINGLE_ITEM_URL_RE.test(href)) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } } -}); - - - /**Blog Landing page*/ -FW.MultiScraper({ -itemType : 'multiple', -detect : FW.Xpath('//div[@id="landing"]//li[contains(@class, "article")]'), -choices : { - titles : FW.Xpath('//li[contains(@class, "article")]/a').text().trim(), - urls : FW.Xpath('//li[contains(@class, "article")]/a').key("href").text() + +async function scrape(doc, url = doc.location.href) { + let type = detectWeb(doc, url); + let translator = Zotero.loadTranslator('web'); + // Embedded Metadata + translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); + translator.setDocument(doc); + + translator.setHandler('itemDone', (_obj, item) => { + item.publicationTitle = "The Atlantic"; + item.libraryCatalog = "The Atlantic"; + + // get rid of trailing "- The Atlantic" in some titles + item.title = item.title.replace(/\s+-\s+The Atlantic$/i, ""); + + // tags from EM is rarely helpful for The Atlantic; they're from the + // meta[name='keywords'] tags, and are either redundant with "Section" + // (in the extra) or spam. + item.tags = []; + + // fix multiple authors; metadata give us one comma-separated string + item.creators = []; + // latter selector for legacy layout + for (let element of doc.querySelectorAll('#byline a[href*="/author/"], .byline span[itemprop="author"]')) { + item.creators.push( + ZU.cleanAuthor( + ZU.trimInternal(element.textContent), + "author" + ) + ); + } + + // keep only the date part of the datetime + if (item.date) { + item.date = ZU.strToISO(item.date); + } + + if (type === "magazineArticle") { + let issueTitle = text(doc, 'div[class*="ArticleMagazineIssueNav_title"]') + .replace(/\s+Issue$/i, ""); + if (issueTitle) { + if (!item.extra) { + item.extra = ""; + } + else { + item.extra += "\n"; + } + item.extra += `Volume Title: ${issueTitle}`; + } + + item.ISSN = "2151-9463"; + } + else if (type === "podcast") { + item.seriesTitle = text(doc, "#rubric"); + let podcasters = []; // plain name strings, "First Last" + for (let creator of item.creators) { + creator.creatorType = "podcaster"; + podcasters.push([creator.firstName, creator.lastName].join(" ")); + } + for (let creator of getPodcastCreators(doc)) { + if (!podcasters.includes(creator)) { + item.creators.push(ZU.cleanAuthor(creator, "guest")); + } + } + } + + item.complete(); + }); + + let em = await translator.getTranslatorObject(); + em.itemType = type || "webpage"; + await em.doWeb(doc, url); +} + +// Get podcast participants from the transcript (paragraph-leading text in +// <strong>). The assumption is that the first mention of anyone is the +// person's full name, and that the further mentions of any particular person +// is a substring of the full name, and they're used consistently in the +// transcripts. +function getPodcastCreators(doc) { + // transcript paragraphs + let containers = doc.querySelectorAll('p[class*="ArticleParagraph_root"], div[class*="ArticleLegacyHtml_root"]'); + + let namesInQuotes = new Set(); + let creatorNames = []; + let skip = new Set(); + Z.debug(`total paragraphs: ${containers.length}`); + let i = 0; // NOTE: debug only + for (let containerElement of containers) { + // Contributor name in bold (<strong> tag), leading a paragraph in the + // transcript + let element = containerElement.querySelector("strong"); + + // if no <strong> tag, or if the first <strong> tag contains all the + // text of the paragraph or div, it's not a name string. + if (!element) continue; + + let elementText = element.textContent; + let nameString = ZU.trimInternal(elementText).replace(/\s*:$/, ""); + if (skip.has(nameString) || !nameString) continue; + if (elementText === containerElement.textContent) continue; + + // Quotes may contain participant's first mentions but also irrelevant + // names (e.g. from newsreels) for context. Stash these names for + // further processing. + if (element.querySelector("em")) { + namesInQuotes.add(nameString); + continue; + } + + i++; // NOTE: debug only + // only if a name string is a not a substring of any other name we've + // kept, add it to the array of name we keep (creatorNames) + // NOTE: nested substring test + if (!creatorNames.filter(keptName => keptName.includes(nameString)).length) { + creatorNames.push(nameString); // keep this string + } + skip.add(nameString); // skip further appearances + } + + Z.debug(`names contributing to substring test cost ${i}`); + // Process names in quoted content (text lines in <em> tags). Take a string + // X from quotes, and check 1) if no kept creator name is a substring of X, + // X is discarded because it's irrelevant; 2) if a string Y from the kept + // names is a substring of X, Y is discarded, X is kept. + Z.debug(`kept names from main paragraphs ${creatorNames.length}`); + creatorNames = new Set(creatorNames); + i = 0; // NOTE: debug only + Z.debug(`names from quotation paragraphs ${namesInQuotes.size}`); + for (let str of namesInQuotes) { + if (skip.has(str)) continue; + i++; // NOTE: debug only + + let keepStr = false; + for (let keptName of creatorNames) { // NOTE: nested substring test + if (str.includes(keptName)) { + creatorNames.delete(keptName); + skip.add(keptName); + keepStr = true; + } + } + if (keepStr) { + creatorNames.add(str); + } + skip.add(str); + } + Z.debug(`names from quotation contributing to substring test cost ${i}`); + + return creatorNames; } -}); - + /** BEGIN TEST CASES **/ var testCases = [ { @@ -105,7 +251,7 @@ var testCases = [ "url": "https://www.theatlantic.com/politics/archive/2011/06/jon-stewart-challenges-fox-news-to-correct-its-errors/240900/", "items": [ { - "itemType": "magazineArticle", + "itemType": "blogPost", "title": "Jon Stewart Challenges Fox News to Correct Its Errors", "creators": [ { @@ -114,15 +260,14 @@ var testCases = [ "creatorType": "author" } ], - "date": "Jun 23, 2011", - "ISSN": "1072-7825", + "date": "2011-06-23", "abstractNote": "In the same segment, the comedian apologized for saying its viewers are always found to be the most misinformed", - "libraryCatalog": "The Atlantic", - "publicationTitle": "The Atlantic", + "blogTitle": "The Atlantic", + "language": "en", "url": "https://www.theatlantic.com/politics/archive/2011/06/jon-stewart-challenges-fox-news-to-correct-its-errors/240900/", "attachments": [ { - "title": "The Atlantic Snapshot", + "title": "Snapshot", "mimeType": "text/html" } ], @@ -146,15 +291,17 @@ var testCases = [ "creatorType": "author" } ], - "date": "July/August 2011 Issue", - "ISSN": "1072-7825", + "date": "2011-06-07", + "ISSN": "2151-9463", "abstractNote": "How a German scientist is using test data to revolutionize global learning", + "extra": "Volume Title: July/August 2011", + "language": "en", "libraryCatalog": "The Atlantic", "publicationTitle": "The Atlantic", "url": "https://www.theatlantic.com/magazine/archive/2011/07/the-worlds-schoolmaster/308532/", "attachments": [ { - "title": "The Atlantic Snapshot", + "title": "Snapshot", "mimeType": "text/html" } ], @@ -175,7 +322,7 @@ var testCases = [ "url": "https://www.theatlantic.com/health/archive/2014/03/the-toxins-that-threaten-our-brains/284466/", "items": [ { - "itemType": "magazineArticle", + "itemType": "blogPost", "title": "The Toxins That Threaten Our Brains", "creators": [ { @@ -184,15 +331,197 @@ var testCases = [ "creatorType": "author" } ], - "date": "Mar 18, 2014", - "ISSN": "1072-7825", + "date": "2014-03-18", "abstractNote": "Leading scientists recently identified a dozen chemicals as being responsible for widespread behavioral and cognitive problems. But the scope of the chemical dangers in our environment is likely even greater. Why children and the poor are most susceptible to neurotoxic exposure that may be costing the U.S. billions of dollars and immeasurable peace of mind.", - "libraryCatalog": "The Atlantic", - "publicationTitle": "The Atlantic", + "blogTitle": "The Atlantic", + "language": "en", "url": "https://www.theatlantic.com/health/archive/2014/03/the-toxins-that-threaten-our-brains/284466/", "attachments": [ { - "title": "The Atlantic Snapshot", + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.theatlantic.com/podcasts/archive/2023/09/ba-286-covid-variant-future/675248/", + "items": [ + { + "itemType": "podcast", + "title": "Our First ‘Nonemergency’ COVID Season", + "creators": [ + { + "firstName": "Hanna", + "lastName": "Rosin", + "creatorType": "podcaster" + }, + { + "firstName": "Katie", + "lastName": "Wu", + "creatorType": "guest" + }, + { + "firstName": "Sarah", + "lastName": "Zhang", + "creatorType": "guest" + } + ], + "abstractNote": "A new wave. A new variant. A new vaccine. Do we know COVID’s annual pattern yet?", + "language": "en", + "seriesTitle": "Radio Atlantic", + "url": "https://www.theatlantic.com/podcasts/archive/2023/09/ba-286-covid-variant-future/675248/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.theatlantic.com/podcasts/archive/2023/08/trans-texas/675188/", + "items": [ + { + "itemType": "podcast", + "title": "When the State Has a Problem With Your Identity", + "creators": [ + { + "firstName": "Hanna", + "lastName": "Rosin", + "creatorType": "podcaster" + }, + { + "firstName": "Ethan", + "lastName": "Brooks", + "creatorType": "podcaster" + }, + { + "firstName": "", + "lastName": "Teenager", + "creatorType": "guest" + }, + { + "firstName": "", + "lastName": "Mom", + "creatorType": "guest" + }, + { + "firstName": "", + "lastName": "Dad", + "creatorType": "guest" + } + ], + "abstractNote": "Inside one family’s decision to move from Texas to California to protect their transgender teenager", + "language": "en", + "seriesTitle": "Radio Atlantic", + "url": "https://www.theatlantic.com/podcasts/archive/2023/08/trans-texas/675188/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.theatlantic.com/podcasts/archive/2023/06/buying-house-with-friends-family/674343/", + "items": [ + { + "itemType": "podcast", + "title": "How to Talk to People: What Makes a House a Home", + "creators": [ + { + "firstName": "Julie", + "lastName": "Beck", + "creatorType": "podcaster" + }, + { + "firstName": "Rebecca", + "lastName": "Rashid", + "creatorType": "podcaster" + }, + { + "firstName": "Deborah", + "lastName": "Tepley", + "creatorType": "guest" + }, + { + "firstName": "Bethany", + "lastName": "Fleming", + "creatorType": "guest" + }, + { + "firstName": "Luke", + "lastName": "Jackson", + "creatorType": "guest" + }, + { + "firstName": "T. J.", + "lastName": "Fleming", + "creatorType": "guest" + } + ], + "abstractNote": "Two married couples who bought a home together have found that expanding their household led to a deeper sense of community.", + "language": "en", + "seriesTitle": "Podcasts", + "shortTitle": "How to Talk to People", + "url": "https://www.theatlantic.com/podcasts/archive/2023/06/buying-house-with-friends-family/674343/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.theatlantic.com/projects/new-rules/", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.theatlantic.com/projects/ideas-2010/archive/2010/06/reading-writing-and-thinking-online-an-interview-with-alan-jacobs/57807/", + "items": [ + { + "itemType": "blogPost", + "title": "Reading, Writing, and Thinking Online: An Interview With Alan Jacobs", + "creators": [ + { + "firstName": "Conor", + "lastName": "Friedersdorf", + "creatorType": "author" + } + ], + "date": "2010-06-08", + "abstractNote": "An accomplished author, essayist and academic on how he successfully navigates the Web, why English students should be forced to recite poems from memory, and why it's a bad idea to read your child Goodnight Moon on a Kindle.", + "blogTitle": "The Atlantic", + "language": "en", + "shortTitle": "Reading, Writing, and Thinking Online", + "url": "https://www.theatlantic.com/projects/ideas-2010/archive/2010/06/reading-writing-and-thinking-online-an-interview-with-alan-jacobs/57807/", + "attachments": [ + { + "title": "Snapshot", "mimeType": "text/html" } ], @@ -203,4 +532,4 @@ var testCases = [ ] } ] -/** END TEST CASES **/ \ No newline at end of file +/** END TEST CASES **/ diff --git a/The New York Review of Books.js b/The New York Review of Books.js index ae2d30ba84d..6a14252d553 100644 --- a/The New York Review of Books.js +++ b/The New York Review of Books.js @@ -3,19 +3,19 @@ "label": "The New York Review of Books", "creator": "Philipp Zumstein", "target": "^https?://www\\.nybooks\\.com/", - "minVersion": "3.0", + "minVersion": "6.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2017-07-02 14:40:20" + "lastUpdated": "2023-08-03 11:23:30" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2017 Philipp Zumstein + Copyright © 2017 Philipp Zumstein and contributors This file is part of Zotero. @@ -37,80 +37,207 @@ function detectWeb(doc, url) { - if (url.indexOf('/articles/')>-1) { + let path = new URL(url).pathname; + if (/^\/articles\//.test(path)) { return "magazineArticle"; - } else if (url.indexOf('/daily/')>-1) { + } + if (/^\/(online|daily)\//.test(path)) { return "blogPost"; - } else if (getSearchResults(doc, true)) { + } + + if (/^\/search\//.test(path)) { // search page + let searchResultRoot = doc.querySelector("#root"); + if (searchResultRoot) { + Z.monitorDOMChanges(searchResultRoot); + } + } + + if (getSearchResults(doc, true)) { return "multiple"; } + return false; } function getSearchResults(doc, checkOnly) { var items = {}; var found = false; - var rows = ZU.xpath(doc, '//h2/a|//h3/a'); - for (var i=0; i<rows.length; i++) { - var href = rows[i].href; - var title = ZU.trimInternal(rows[i].textContent); + var rows = doc.querySelectorAll(".container .row, .sui-result"); + for (let row of rows) { + let href = attr(row, "h4 > a, .sui-result__title-link", "href"); + let title = ZU.trimInternal(text(row, "h4 > a, .sui-result__title-link")); if (!href || !title) continue; if (checkOnly) return true; found = true; - items[href] = title; + // Do a bit of disambiguition using author names; try this: + // https://www.nybooks.com/search?q=star + let author = ZU.trimInternal(text(row, "div:first-child")); + items[href] = author ? `${title} (${author.slice(0, 40).trim()})` : title; } return found ? items : false; } - -function doWeb(doc, url) { - if (detectWeb(doc, url) == "multiple") { - Zotero.selectItems(getSearchResults(doc, false), function (items) { - if (!items) { - return true; - } - var articles = []; - for (var i in items) { - articles.push(i); - } - ZU.processDocuments(articles, scrape); - }); - } else { - scrape(doc, url); +async function doWeb(doc, url) { + if (detectWeb(doc, url) == 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); } } - -function scrape(doc, url) { - var type = (url.indexOf('/articles/')>-1) ? "magazineArticle" : "blogPost"; +async function scrape(doc, url = doc.location.href) { + var type = detectWeb(doc, url); var translator = Zotero.loadTranslator('web'); // Embedded Metadata translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48'); - //translator.setDocument(doc); - + + // Work around the synchronicity restruction of handler callback; + // Do necessary computation involving async function out of the callback. + // Date; also useful for retrieving volume and number + let dateArray = urlToDateArray(url); + + // Get { volume, issue } from the article's issue page asynchronously + let itemStub; + if (type === "magazineArticle" && dateArray) { + itemStub = await getIssue(dateArray); + } + translator.setHandler('itemDone', function (obj, item) { - if (type=="magazineArticle") { + item.publicationTitle = "The New York Review of Books"; + Object.assign(item, itemStub); // absorb volume/number info + + if (type == "magazineArticle") { item.ISSN = "0028-7504"; } - if (!item.date) { - var date = ZU.xpathText(doc, '//article//time'); - if (date) { - item.date = ZU.strToISO(date.replace('Issue', '')); + + if (!item.date && dateArray) { + item.date = dateArray.join("-"); + } + + // Remove noise trailing the "pipe" character + let cleanTitle = text(doc, "header.article-header h1"); + if (!cleanTitle) { + cleanTitle = item.title.split("|")[0].trim(); + } + item.title = cleanTitle || item.title; + + // Remove author of editorials + if (item.creators && item.creators.length === 1) { + let authorString = creatorToString(item.creators[0]); + if (authorString.toLowerCase() === "the editors") { + item.creators = []; + } + } + + // Find any translators and fix their type (the EM doesn't mark them as + // translators) + let authorByline = ZU.trimInternal(text(doc, "header .author")); + // Split off any ", translated by ..." + let maybeTranslator = authorByline.split(/\btranslat\S+\s+by\s+/i)[1]; + if (maybeTranslator) { + // Iterate over creators and check if the full name appears in the + // "translated by ..." part + for (let author of item.creators) { + let authorString = creatorToString(author); + if (maybeTranslator.includes(authorString)) { + author.creatorType = "translator"; + } + } + } + + // Add reviewed authors (only for "author" authors, not including + // works without leading author but having editor/translator) + let reviewed = doc.querySelectorAll(".review-items .attribution"); + for (let bylineElem of reviewed) { + let byline = ZU.trimInternal(bylineElem.textContent.trim()); + for (let author of cleanReviewedByline(byline)) { + item.creators.push(ZU.cleanAuthor(author, "reviewedAuthor")); } } + item.complete(); }); - translator.getTranslatorObject(function(trans) { - trans.itemType = type; - trans.doWeb(doc, url); - }); + let em = await translator.getTranslatorObject(); + em.itemType = type; + await em.doWeb(doc, url); +} + +// Parse URL path and extract the year, month, and day as array. +function urlToDateArray(url) { + let path = new URL(url).pathname; + let m = path.match(/\/(\d{4})\/(\d{1,2})\/(\d{1,2})\//); // yyyy mm dd + return m && [m[1], m[2], m[3]]; +} + +// Given the date-array of the issue, find the volume and number by requesting +// the issue's page and parse the metadata. This function is memoized. +var ISSUE_CACHE = {}; +async function getIssue(dateArray) { + let key = dateArray.join("/"); + if (ISSUE_CACHE[key]) { + return ISSUE_CACHE[key]; + } + let issueURL = `https://www.nybooks.com/issues/${key}/`; + let issuePageDoc = await requestDocument(issueURL); + + // Use LD-JSON to retrieve the breadcrumb containing volume / issue info + let linkedData = text(issuePageDoc, "script[type='application/ld+json']"); + if (linkedData) { + let issueInfo = JSON.parse(linkedData); + let breadcrumbs = issueInfo["@graph"].filter(x => x["@type"] === "BreadcrumbList"); + if (breadcrumbs.length) { + breadcrumbs = breadcrumbs[0].itemListElement || []; + } + for (let bcItem of breadcrumbs) { + let m = bcItem.name.match(/^volume (\d+), (?:issue|number) (\d+)/i); + if (m) { + let result = { volume: m[1], issue: m[2] }; + ISSUE_CACHE[key] = result; + return result; + } + } + } + return null; +} + +// Turn creator object to string in "First Last" +function creatorToString(creator) { + return [creator.firstName, creator.lastName].filter(Boolean).join(" "); +} + +// For any single attribution line of the work being reviewed, return the name +// of authors (excluding translators, editors, etc.) +function cleanReviewedByline(byline) { + let authors = []; + let m = byline.match(/^by\s+(.+)/i); + + if (!m) { // complex byline, without first "author"; ignore + return authors; + } + + byline = m[1]; // remove leading "by" + for (let item of byline.split(/(,|\band\b)/)) { + item = item.trim(); + if (item.startsWith("translat")) { + break; + } + if (item && item !== "," && item !== "and") { + authors.push(item); + } + } + return authors; } /** BEGIN TEST CASES **/ var testCases = [ { "type": "web", - "url": "http://www.nybooks.com/articles/2011/12/08/zuccotti-park-what-future/", + "url": "https://www.nybooks.com/articles/2011/12/08/zuccotti-park-what-future/", "items": [ { "itemType": "magazineArticle", @@ -125,13 +252,17 @@ var testCases = [ "date": "2011-12-08", "ISSN": "0028-7504", "abstractNote": "For weeks, organizers had demonstrated enormous skill in keeping the occupation going, steadily expanding while outfoxing Mayor Bloomberg in his attempts to evict them. But what end did it serve if their status as ethical defenders of the 99 percent was being damaged? It was, after all, their major asset. The complicated logistics of holding the park (and providing food, clothing, and warmth for a floating army of hundreds) was draining resources and forcing the most talented activists to narrow their focus to matters of mere physical survival.", + "issue": "19", + "language": "en", "libraryCatalog": "www.nybooks.com", "publicationTitle": "The New York Review of Books", "shortTitle": "Zuccotti Park", - "url": "http://www.nybooks.com/articles/2011/12/08/zuccotti-park-what-future/", + "url": "https://www.nybooks.com/articles/2011/12/08/zuccotti-park-what-future/", + "volume": "58", "attachments": [ { - "title": "Snapshot" + "title": "Snapshot", + "mimeType": "text/html" } ], "tags": [], @@ -142,12 +273,13 @@ var testCases = [ }, { "type": "web", - "url": "http://www.nybooks.com/search/?s=labor+union", + "url": "https://www.nybooks.com/search/?q=labor+union", + "defer": true, "items": "multiple" }, { "type": "web", - "url": "http://www.nybooks.com/daily/2011/11/16/americas-new-robber-barons/", + "url": "https://www.nybooks.com/online/2011/11/16/americas-new-robber-barons/", "items": [ { "itemType": "blogPost", @@ -160,19 +292,17 @@ var testCases = [ } ], "date": "2011-11-16", - "abstractNote": "With early Tuesday’s abrupt evacuation of Zuccotti Park, the City of New York has managed—for the moment—to dislodge protesters from Wall Street. But it will be much harder to turn attention away from the financial excesses of the very rich—the problems that have given Occupy Wall Street such traction. Data on who is in the top 1 percent of earners further reinforces their point. Here's why.\n\nThough the situation is often described as a problem of inequality, this is not quite the real concern. The issue is runaway incomes at the very top—people earning a million and a half dollars or more according to the most recent data. And much of that runaway income comes from financial investments, stock options, and other special financial benefits available to the exceptionally rich—much of which is taxed at very low capital gains rates. Meanwhile, there has been something closer to stagnation for almost everyone else—including even for many people in the top 20 percent of earners.", + "abstractNote": "With early Tuesday’s abrupt evacuation of Zuccotti Park, the City of New York has managed—for the moment—to dislodge protesters from Wall Street. But it will be much harder to turn attention away from the financial excesses of the very rich—the problems that have given Occupy Wall Street such traction. Data on who is in the top 1 percent of earners further reinforces their point. Here's why. Though the situation is often described as a problem of inequality, this is not quite the real concern. The issue is runaway incomes at the very top—people earning a million and a half dollars or more according to the most recent data. And much of that runaway income comes from financial investments, stock options, and other special financial benefits available to the exceptionally rich—much of which is taxed at very low capital gains rates. Meanwhile, there has been something closer to stagnation for almost everyone else—including even for many people in the top 20 percent of earners.", "blogTitle": "The New York Review of Books", - "url": "http://www.nybooks.com/daily/2011/11/16/americas-new-robber-barons/", + "language": "en", + "url": "https://www.nybooks.com/online/2011/11/16/americas-new-robber-barons/", "attachments": [ { - "title": "Snapshot" + "title": "Snapshot", + "mimeType": "text/html" } ], - "tags": [ - "Occupy Wall Street", - "economics", - "inequality" - ], + "tags": [], "notes": [], "seeAlso": [] } @@ -182,6 +312,156 @@ var testCases = [ "type": "web", "url": "http://www.nybooks.com/issues/2012/03/22/", "items": "multiple" + }, + { + "type": "web", + "url": "https://www.nybooks.com/articles/1983/12/08/the-second-death-of-peron/", + "items": [ + { + "itemType": "magazineArticle", + "title": "The Second Death of Perón?", + "creators": [ + { + "firstName": "Robert", + "lastName": "Cox", + "creatorType": "author" + }, + { + "firstName": "Joseph A.", + "lastName": "Page", + "creatorType": "reviewedAuthor" + }, + { + "firstName": "Lars", + "lastName": "Schoultz", + "creatorType": "reviewedAuthor" + } + ], + "date": "1983-12-08", + "ISSN": "0028-7504", + "abstractNote": "Perón, Perón, how great you are. My general, how much you're worth! As the chorus of the Peronist marching song \"The Peronist Boys\" suggests, there was", + "issue": "19", + "language": "en", + "libraryCatalog": "www.nybooks.com", + "publicationTitle": "The New York Review of Books", + "url": "https://www.nybooks.com/articles/1983/12/08/the-second-death-of-peron/", + "volume": "30", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.nybooks.com/articles/1983/12/22/double-trouble/", + "items": [ + { + "itemType": "magazineArticle", + "title": "Double Trouble", + "creators": [ + { + "firstName": "David", + "lastName": "Bellos", + "creatorType": "translator" + }, + { + "firstName": "Emmanuel Le Roy", + "lastName": "Ladurie", + "creatorType": "author" + }, + { + "firstName": "Natalie Zemon", + "lastName": "Davis", + "creatorType": "reviewedAuthor" + } + ], + "date": "1983-12-22", + "ISSN": "0028-7504", + "abstractNote": "The biographies of peasants and especially the autobiographies of country people are a longstanding problem. We owe to the habits of Protestant", + "issue": "20", + "language": "en", + "libraryCatalog": "www.nybooks.com", + "publicationTitle": "The New York Review of Books", + "url": "https://www.nybooks.com/articles/1983/12/22/double-trouble/", + "volume": "30", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.nybooks.com/articles/2015/02/05/putins-kleptocracy/", + "items": [ + { + "itemType": "magazineArticle", + "title": "‘Putin’s Kleptocracy’", + "creators": [], + "date": "2015-02-05", + "ISSN": "0028-7504", + "abstractNote": "Lucy Komisar has written to us that she has a statement to make about the article by Anne Applebaum, “How He and His Cronies Stole Russia,” in our", + "issue": "2", + "language": "en", + "libraryCatalog": "www.nybooks.com", + "publicationTitle": "The New York Review of Books", + "url": "https://www.nybooks.com/articles/2015/02/05/putins-kleptocracy/", + "volume": "62", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.nybooks.com/online/2012/12/15/our-moloch/", + "items": [ + { + "itemType": "blogPost", + "title": "Our Moloch", + "creators": [ + { + "firstName": "Garry", + "lastName": "Wills", + "creatorType": "author" + } + ], + "date": "2012-12-15", + "abstractNote": "The gun is our Moloch. We sacrifice children to him daily—sometimes, as at Sandy Hook, by directly throwing them into the fire-hose of bullets from our protected private killing machines, sometimes by blighting our children’s lives by the death of a parent, a schoolmate, a teacher, a protector. The gun is not a mere tool, a bit of technology, a political issue, a point of debate. It is an object of reverence.", + "blogTitle": "The New York Review of Books", + "language": "en", + "url": "https://www.nybooks.com/online/2012/12/15/our-moloch/", + "attachments": [ + { + "title": "Snapshot", + "mimeType": "text/html" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] } ] -/** END TEST CASES **/ \ No newline at end of file +/** END TEST CASES **/ diff --git a/Twitter.js b/Twitter.js index 1e42b7fd1d1..d21419f5d92 100644 --- a/Twitter.js +++ b/Twitter.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-02-01 20:17:45" + "lastUpdated": "2023-08-17 18:35:48" } /* @@ -37,7 +37,7 @@ */ -let titleRe = /^(?:\(\d+\) )?(.+) .* Twitter: .([\S\s]+). \/ Twitter/; +let titleRe = /^(?:\(\d+\) )?(.+) .* (?:Twitter|X): .([\S\s]+). \/ (?:Twitter|X)/; function detectWeb(doc, url) { if (url.includes('/status/')) { diff --git a/Web of Science Tagged.js b/Web of Science Tagged.js index 0d5f42ee4a5..3b4e1a5ab68 100644 --- a/Web of Science Tagged.js +++ b/Web of Science Tagged.js @@ -1,20 +1,20 @@ { "translatorID": "594ebe3c-90a0-4830-83bc-9502825a6810", "label": "Web of Science Tagged", - "creator": "Michael Berkowitz, Avram Lyon", + "creator": "Michael Berkowitz, Avram Lyon, and contributors", "target": "txt", "minVersion": "2.1", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 1, - "lastUpdated": "2021-07-21 02:48:00" + "lastUpdated": "2023-07-17 03:09:44" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2015-2021 Michael Berkowitz, Avram Lyon + Copyright © 2015-2021 Michael Berkowitz, Avram Lyon, and contributors. This file is part of Zotero. @@ -34,251 +34,474 @@ ***** END LICENSE BLOCK ***** */ +// Lookup tables +var ITEM_TYPES = { + // PT values + J: "journalArticle", + S: "bookSection", // Not sure + P: "patent", + B: "book", + // DT overrides; not including anything already covered by the above. + // TODO: Add more implementations for DT values as needed. + "PROCEEDINGS PAPER": "conferencePaper", +}; + +var FIELD_MAP = { + AE: "assignee", + AB: "abstractNote", + AR: "pages", // article number + AW: "url", + BN: "ISBN", + BP: "pages", // start page + CE: "edition", + CL: "place", // conference location + CT: "conferenceName", + // NOTE: CY is conference date, not "issued" (published) date + DI: "DOI", + FN: "libraryCatalog", // almost always the database name + // XXX: what is "GT: Time"? + IO: "issuingAuthority", + IS: "issue", + JI: "journalAbbreviation", + LA: "language", + PA: "place", + PC: "country", // patent country + PD: "date", + PG: "numPages", + PI: "place", // publisher city + PN: "patentNumber", + PS: "pages", + PU: "publisher", + PV: "place", // place of publication + PY: "date", // publication year + UR: "url", + VL: "volume", + VN: "versionNumber", // NOTE: "VR" is the version of the export format + // Title fields, + SE: "seriesTitle", + SO: "publicationTitle", + TI: "title", +}; + +// Translator detect/do functions + function detectImport() { - var line; - var i = 0; - while ((line = Zotero.read()) !== false) { - line = line.replace(/^\s+/, ""); - if (line != "") { - if (line.substr(0, 4).match(/^PT [A-Z]/)) { + // If we don't find item type (PT or DT) within first 10 non-empty lines, + // return false; see also RIS.js. + let line; + let i = 0; + while ((line = Zotero.read()) !== false && i < 10) { + let lineData = splitLine(line); + if (lineData) { + i += 1; + if (["PT", "DT"].includes(lineData[0])) { return true; - } else { - if (i++ > 3) { - return false; - } } } } + return false; +} + +function doImport() { + let map = new ItemMap(); + + let line; + while (!map.terminate && ((line = Z.read()) !== false)) { + map.scanLine(line); + } + // Try saving any leftover fields as an item; this can happen when the last + // record lacks ER or EF. In that case, we shouldn't throw away the data + // simply because the file didn't properly end. + map.save(); + return false; +} + +// Utilities + +/** + * Utility for mapping tagged data to item fields + * + * @constructor + */ +function ItemMap() { + // Hold the property => value map for an item. + this.records = new Map(); + // Cursor to current property + this.currentKey = null; + this.terminate = false; } -function processTag(item, field, content) { - var map = { - "J": "journalArticle", - "S": "bookSection", // Not sure - "P": "patent", - "B": "book" - }; - if (field == "PT") { - item.itemType = map[content]; - if (item.itemType === undefined) { - item.itemType = "journalArticle"; - Zotero.debug("Unknown type: " + content); +ItemMap.prototype = { + + /** + * Validate a line and push it into the item map record if it's valid data + * line, or do an action if it is one of the special tags (ER and EF). + * + * This function is strictly concerned with the form of the lines and + * converting the line data to key-value pairs. It does not apply any + * semantics to the data. + * + * @param {string} line + * @throws {Error} When line fails validation + */ + scanLine: function (line) { + let lineRecord = splitLine(line); + if (!lineRecord) { + return; } - } else if (field == "DT") { - if (content.trim().toLowerCase() == "proceedings paper") { - item.itemType = "conferencePaper"; + let [head, content] = lineRecord; + + if (head === " ") { // two spaces for line continuation + if (!this.currentKey) { + throw new Error(`Dangling line continuation; the rest of line is ${content}`); + } + + let currentValueArray = this.records.get(this.currentKey); + + if (!Array.isArray(currentValueArray)) { + // Continued line can be traced to a tag but the tag's field is + // not initialized. This should not happen and is an internal + // error. + throw new Error(`Unexpected uninitialized field at ${this.currentKey} found while handling line continuation`); + } + + // Add the continued line, if non-empty, into the container for the + // field value + if (content) { + currentValueArray.push(content); + } } - } else if ((field == "AF" || field == "AU")) { - //Z.debug("author: " + content); - const authors = content.split("\n"); - for (var i=0; i<authors.length; i++) { - var author = authors[i]; - author = author.replace(/\s+\(.*/, ''); - item.creators[0][field].push(ZU.cleanAuthor(author, "author", author.match(/,/))); + else { // Not line continuation; a new tag begins. + if (head === "ER") { + // End of Record; Save this item and reset for any subsequent + // items. + this.save(); + this.reset(); + return; + } + + if (head === "EF") { + // End of File; Simply signal the end of processing. + this.terminate = true; + return; + } + + // Double check if there is duplicate tag + if (this.records.has(head)) { + // TODO: should it throw? + Z.debug(`Warning: duplicate tag ${head}; new input field value is ${content}; old value was ${this.records.get(head).toString()}`); + } + + // Initialize the tag field with a new array to accommodate + // continued lines + this.records.set(head, content ? [content] : []); + this.currentKey = head; } - } else if ((field == "BE")) { - //Z.debug(content); - const authors = content.split("\n"); - for (var i=0; i<authors.length; i++) { - var author = authors[i]; - item.creators[1].push(ZU.cleanAuthor(author, "editor", author.match(/,/))); + }, + + /** + * Create a new Zotero item, populate it using this item map after + * normalization, and complete the Zotero item. + */ + save: function () { + this.normalize(); + + // Pop the type string from the normalized record + let type = this.records.get("DT"); + this.records.delete("DT"); + + let item = new Z.Item(type); + let extra = []; // temporary array for holding lines in the extra + let tagMissed = 0; // number of tags unhandled + + for (let [wosTag, tagValueArray] of this.records) { + // The array content concatenated into a single string + let tagValueString = ZU.trimInternal(tagValueArray.join(" ")); + + switch (wosTag) { + // Main authors. After normalization, AF and AU cannot both + // exist. + case "AF": + case "AU": + addCreator(item, tagValueArray, + type === "patent" ? "inventor" : "author"); + break; + // Other types of creators + case "BE": // Book editor + case "ED": // Editors + addCreator(item, tagValueArray, "editor"); + break; + case "TR": + addCreator(item, tagValueArray, "translator"); + break; + case "AA": // Additional Authors + addCreator(item, tagValueArray, "contributor"); + break; + + // Keywords, ontology terms, etc. as item tags + case "BD": // broad terms, like DE below + case "DE": // author-defined keywords + case "ID": // keywords + case "IP": // patent category + case "MC": // Major Concepts or Derwent Manual Code(s) + case "MQ": // methods, supplies + case "OR": // organism descriptors + item.tags.push(...tagValueString.split("; ")); + break; + + // Additional information that becomes lines in the extra field + case "NO": + extra.push(`Comments, Corrections, Erratum: ${tagValueString}`); + break; + case "NT": + extra.push(`Notes: ${tagValueString}`); + break; + case "UT": + extra.push(`Web of Science ID: ${tagValueString}`); + break; + + // ISSN + case "SN": + item.ISSN = tagValueArray.join(", ") || undefined; + break; + + // Title fields (often in all-caps) are turned into Title Case + // if the pref "capitalizeTitles" is true (default false) + case "SE": + case "SO": + case "TI": + tagValueString = selectiveTitleCase(tagValueString); + item[FIELD_MAP[wosTag]] = tagValueString; + break; + // The following non-title fields are converted to Title Case + case "AE": // patent assignee + case "CT": // conference name + case "PU": // publisher + tagValueString = selectiveTitleCase(tagValueString, true/* force */); + /* NOTE: FALL THROUGH */ + default: + { + let itemField = FIELD_MAP[wosTag]; + if (!itemField) { // unknown tag + Z.debug(`Unhandled tag ${wosTag} => ${tagValueArray}`); + tagMissed += 1; + } + else { + item[itemField] = tagValueString; + } + } + } // bottom of the switch statement } - } else if (field == "TI") { - content = content.replace(/\s\s+/g, " "); - item.title = content; - } else if (field == "JI") { - item.journalAbbreviation = content; - } else if (field == "SO") { - item.publicationTitle = content; - } else if (field == "SN") { - item.ISSN = content; - } else if (field == "BN") { - item.ISBN = content; - } else if (field == "PD" || field == "PY") { - if (item.date) { - item.date += " " + content; - } else { - item.date = content; + + if (tagMissed === this.records.size) { // item is not populated + return; } - var year = item.date.match(/\d{4}/); - // If we have a double year, eliminate one - if (year && item.date.replace(year[0],"").includes(year[0])) - item.date = item.date.replace(year[0],""); - } else if (field == "VL") { - item.volume = content; - } else if (field == "IS") { - item.issue = content; - } else if (field == "UT") { - item.extra += content; - } else if (field == "BP" || field == "PS") { // not sure why this varies - item.pages = content; - } else if (field == "EP") { - item.pages += "-" + content; - } else if (field == "AR") { - //save articleNumber - we're going to use that where we don't have pages & discard later on - item.articleNumber = content; ; - } else if (field == "AB") { - content = content.replace(/\s\s+/g, " "); - item.abstractNote = content; - } else if (field == "PI" || field == "C1") { - item.place = content; - } else if (field == "LA") { - item.language = content; - } else if (field == "PU") { - item.publisher = content; - // Patent stuff - } else if (field == "DG") { - item.issueDate = content; - } else if (field == "PN") { - item.patentNumber = content; - } else if (field == "AE") { - item.assignee = content; - } else if (field == "PL") { // not sure... - item.priorityNumber = content; - } else if (field == "PC") { // use for patents - item.country = content; - // A whole mess of tags - } else if (field == "DE" || field == "BD" - || field == "OR" || field == "ID" - || field == "MC" || field == "MQ") { - item.tags = item.tags.concat(content.split(";")); - } else if (field == "DI") { - item.DOI = content; - } else { - Zotero.debug("Discarding: " + field + " => "+content); - } -} -function completeItem(item) { - var i; - var creators = []; - // If we have full names, drop the short ones - if (item.creators[0]["AF"].length) { - creators = item.creators[0]["AF"]; - } else { - creators = item.creators[0]["AU"]; - } - // Add other creators - if (item.creators[1]) - item.creators = creators.concat(item.creators[1]); - else - item.creators = creators; - - // If we have a patent, change author to inventor - if (item.itemType == "patent") { - for (i in item.creators) { - if (item.creators[i].creatorType == "author") { - item.creators[i].creatorType = "inventor"; + if (extra.length) { + item.extra = extra.join("\n"); + } + + item.complete(); + }, + + /** + * Reset the internal state of the item map + */ + reset: function () { + this.records.clear(); + this.currentKey = null; + this.terminate = false; + }, + + /** + * Normalize the content of internal records. This must be called only + * after all the fields of an item is populated. + */ + normalize: function () { + let r = this.records; // for ergonomics + + // Normalize type. Turn the field value into a Zotero item-type string + // for convenience because it is always used in a special way, unlike + // other tags. DT takes precedence over PT. + // NOTE: After we identify the Zotero type, DT is set and PT deleted. + let wosType = r.get("DT"); // If DT present, begin with it + let zoteroType = wosToZoteroType(wosType, "DT"); + if (!zoteroType) { + wosType = r.get("PT"); // Try PT next if DT not useable + zoteroType = wosToZoteroType(wosType, "PT"); + } + if (!zoteroType) { + Z.debug("Warning: No type found for item; falling back to journal article"); + zoteroType = "journalArticle"; + } + // Delete PT and set DT to Zotero type + r.delete("PT"); + r.set("DT", zoteroType); + + // Authors. Use AF in preference to AU. + if (r.has("AF") && r.has("AU")) { + r.delete("AU"); + } + + // Normalize pages. If page range present, use it. + if (r.has("PS")) { + r.delete("BP"); // begin page + r.delete("EP"); // end page + } + else { // no page range + let begin = r.get("BP"); + let end = r.get("EP"); + // If BP exists, try construct a page range like "begin-end" if EP + // exists, or without the "-end" suffix if EP is missing. + if (begin) { + let suffix = (end && end[0]) ? `-${end[0]}` : ""; + r.set("PS", [`${begin[0]}${suffix}`]); + r.delete("BP"); + r.delete("EP"); } } + + // Most electronic journals use article number (AR), but if both + // article number and pages present, ignore article number. + if (r.has("AR") && (r.has("PS") || r.has("BP"))) { + r.delete("AR"); + } + + // Normalize dates. PY refers to the year, and PD is the more detailed + // date, like month-day ("JAN 1"), month ("JAN"), season ("SPR"), + // possibly with redundant year. + if (r.has("PY")) { // year + if (!r.has("PD")) { // but without PD + Z.debug(`Using PY ${r.get("PY")} verbatim as date`); + r.set("PD", r.get("PY")); + } + else { + let year = r.get("PY")[0]; + let pd = r.get("PD")[0]; + if (!pd.includes(year)) { // PD doesn't have redundant year + Z.debug(`Adding PY ${year} to PD ${pd}`); + pd += ` ${year}`; + r.set("PD", [pd]); + } + } + // We have put whatever date details we have into PD, and PY is now + // redundant + r.delete("PY"); + } + + // Publisher address; don't use full address if PI (city) is available + if (r.has("PI")) { + let city = r.get("PI")[0]; + r.set("PI", [ZU.capitalizeTitle(city, true/* force */)]); + r.delete("PA"); + } + + // ISSN; consolidate EI (eISSN) into SN + let issn = [...(r.get("SN") || []), ...(r.get("EI") || [])] + .filter(Boolean); + if (issn.length) { + r.set("SN", issn); + r.delete("EI"); + } + }, +}; + +// split line into an array of [tag, tag value] (the latter optional). +function splitLine(line) { + // First trim line end and strip the line of BOM (U+FEFF) if any, as + // show in a test case + // TODO: Use trimEnd() once Z6 support is dropped + line = line.replace(/\s*$/, ''); + line = line.replace(/\uFEFF/g, ""); + + // skip empty line + if (!line.length) { + return null; } - - if (item.articleNumber){ - if (!item.pages) item.pages = item.articleNumber; - delete item.articleNumber; - } - - // Fix caps, trim in various places - for (i in item.tags) { - item.tags[i] = item.tags[i].trim(); - if (item.tags[i].toUpperCase() == item.tags[i]) - item.tags[i]=item.tags[i].toLowerCase(); + + // Split line into tag and "content" that comes after the tag. + // NOTE: When the file doesn't use explicit line-continuation (three + // spaces), there's ambiguity when the line meant to be continued data + // happens to match the pattern of a tag-value line. + let m = line.match(/^( {2}|[A-Z][A-Z0-9])( .+)?$/); + if (!m) { + // Z.debug(`Possible continued-line without explicit line continuation: ${line.slice(0, 5)}...`); + return [" "/* two spaces */, line]; } - - var toFix = ["publisher", "publicationTitle", "place"]; - for (i in toFix) { - var field = toFix[i]; - if (item[field] && item[field].toUpperCase() == item[field]) - item[field]=ZU.capitalizeTitle(item[field].toLowerCase(),true); + + return [m[1]/* tag */, m[2] && m[2].trim()/* tag content, or undefined */]; +} + +// Convenience functions + +function addCreator(item, authorArray, authorType) { + for (let author of authorArray) { + item.creators.push(stringToCreator(author, authorType)); } +} + +function wosToZoteroType(wosType, debugTag) { + let zoteroType; + if (wosType) { + // case-normalized WoS type string + let wosTypeNormalized = wosType[0].toUpperCase(); - item.complete(); + if (ITEM_TYPES[wosTypeNormalized]) { // value is understood + Z.debug(`Using ${debugTag} ${wosType} for item type`); + zoteroType = ITEM_TYPES[wosTypeNormalized]; + } + else { + Z.debug(`Ignoring unimplemented ${debugTag} value ${wosType}`); + } + } + return zoteroType; } -function doImport(text) { - var tag = data = false; - var debugBuffer = ''; //in case nothing is found - var linesRead = 0, bufferMax = 100; - var line = Zotero.read(); - // first valid line is type - while (line !== false && line.replace(/^\s+/, "").substr(0, 6).search(/^PT [A-Z]/) == -1) { - if (linesRead < bufferMax) debugBuffer += line + '\n'; - linesRead++; - line = Zotero.read(); +function stringToCreator(author, type) { + // Strip any parenthesized text + author = author.replace(/\(.*\)/g, ""); + if (!author.includes(",")) { // as "LAST F", rather than "Last, F" + author = author.replace(" ", ", "); // replace first space } + return ZU.cleanAuthor(author, type, true/* useComma */); +} - if (line === false) { - Z.debug("No valid data found\n" + - "Read " + linesRead + " lines.\n" + - "Here are the first " + (linesRead<bufferMax?linesRead:bufferMax) + " lines:\n" + - debugBuffer); - return; +// like ZU.capitalizeTitle but mindful of some words that are often encountered +// in conference or publisher names. This is most useful for cleaning all-cap +// fields that are not titles. +function selectiveTitleCase(string, force) { + let allCaps = ["ACM", "AIP", "BMC", "BMJ", "CRC"/* CRC Press */, "IEEE", "JAMA", "MDPI", "SAGE", "USA"]; + let wordForms = { IOP: "IoP", PEERJ: "PeerJ", PLOS: "PLoS" }; + for (let word of allCaps) { + wordForms[word] = word; } - var item = new Zotero.Item(); - var i = 0; - item.creators = [{"AU":[], "AF":[]}, []]; - item.extra = ""; - - - var tag = "PT"; - - var data = line.substr(3); - - var rawLine; - while ((rawLine = Zotero.read()) !== false) { // until EOF - //Z.debug("line: " + rawLine); - let split = rawLine.match(/^([A-Z0-9]{2})\s(?:([^\n]*))?/); - // Force a match for ER - if (rawLine == "ER") split = ["","ER",""]; - if (split) { - // if this line is a tag, take a look at the previous line to map - // its tag - if (tag) { - //Zotero.debug("tag: '"+tag+"'; data: '"+data+"'"); - processTag(item, tag, data); - } + let cleanInput = ZU.trimInternal(string); + let cleanInputArray = cleanInput.split(" "); - // then fetch the tag and data from this line - tag = split[1]; - data = split[2] || ''; - - if (tag == "ER") { // ER signals end of reference - // unset info - tag = data = false; - completeItem(item); - } - if (tag == "PT") { - // new item - item = new Zotero.Item(); - item.creators = [{"AU":[], "AF":[]}, []]; - item.extra = ""; - i++; - } - } else { - // otherwise, assume this is data from the previous line continued - if (tag == "AU" || tag == "AF" || tag == "BE") { - //Z.debug(rawLine); - // preserve line endings for AU fields - data += "\n" + rawLine; - } else if (tag) { - // otherwise, concatenate and avoid extra spaces - if (data[data.length-1] == " " || rawLine[0] == " ") { - data += rawLine; - } else { - data += " "+rawLine; - } - } + let arrayLocations = new Map(); + + for (let [word, form] of Object.entries(wordForms)) { + for (let index of indexOfAll(cleanInputArray, word)) { + arrayLocations.set(index, form); } } - if (tag && tag != "ER") { // save any unprocessed tags - //Zotero.debug(tag); - processTag(item, tag, data); - completeItem(item); + let outputArray = ZU.capitalizeTitle(cleanInput, force).split(" "); + + for (let [index, form] of arrayLocations.entries()) { + outputArray[index] = form; } + return outputArray.join(" "); } +// Search array for searchElement. Returns an array of indices where +// searchElement appears, or empty array if searchElement is missing. +function indexOfAll(array, searchElement) { + return array + .map((elem, index) => (elem === searchElement ? index : null)) + .filter(x => x !== null); +} /** BEGIN TEST CASES **/ var testCases = [ @@ -288,7 +511,7 @@ var testCases = [ "items": [ { "itemType": "journalArticle", - "title": "Anaplerotic Role for Cytosolic Malic Enzyme in Engineered Saccharomyces cerevisiae Strains", + "title": "Anaplerotic Role for Cytosolic Malic Enzyme in Engineered Saccharomyces cerevisiae Strains", "creators": [ { "firstName": "Rintze M.", @@ -311,14 +534,15 @@ var testCases = [ "creatorType": "author" } ], - "date": "FEB 2011", + "date": "FEB 2011", "DOI": "10.1128/AEM.02132-10", "ISSN": "0099-2240", - "abstractNote": "Malic enzyme catalyzes the reversible oxidative decarboxylation of malate to pyruvate and CO(2). The Saccharomyces cerevisiae MAE1 gene encodes a mitochondrial malic enzyme whose proposed physiological roles are related to the oxidative, malate-decarboxylating reaction. Hitherto, the inability of pyruvate carboxylase-negative (Pyc(-)) S. cerevisiae strains to grow on glucose suggested that Mae1p cannot act as a pyruvate-carboxylating, anaplerotic enzyme. In this study, relocation of malic enzyme to the cytosol and creation of thermodynamically favorable conditions for pyruvate carboxylation by metabolic engineering, process design, and adaptive evolution, enabled malic enzyme to act as the sole anaplerotic enzyme in S. cerevisiae. The Escherichia coli NADH-dependent sfcA malic enzyme was expressed in a Pyc(-) S. cerevisiae background. When PDC2, a transcriptional regulator of pyruvate decarboxylase genes, was deleted to increase intracellular pyruvate levels and cells were grown under a CO(2) atmosphere to favor carboxylation, adaptive evolution yielded a strain that grew on glucose (specific growth rate, 0.06 +/- 0.01 h(-1)). Growth of the evolved strain was enabled by a single point mutation (Asp336Gly) that switched the cofactor preference of E. coli malic enzyme from NADH to NADPH. Consistently, cytosolic relocalization of the native Mae1p, which can use both NADH and NADPH, in a pyc1,2 Delta pdc2 Delta strain grown under a CO(2) atmosphere, also enabled slow-growth on glucose. Although growth rates of these strains are still low, the higher ATP efficiency of carboxylation via malic enzyme, compared to the pyruvate carboxylase pathway, may contribute to metabolic engineering of S. cerevisiae for anaerobic, high-yield C(4)-dicarboxylic acid production.", - "extra": "WOS:000286597100004", + "abstractNote": "Malic enzyme catalyzes the reversible oxidative decarboxylation of malate to pyruvate and CO(2). The Saccharomyces cerevisiae MAE1 gene encodes a mitochondrial malic enzyme whose proposed physiological roles are related to the oxidative, malate-decarboxylating reaction. Hitherto, the inability of pyruvate carboxylase-negative (Pyc(-)) S. cerevisiae strains to grow on glucose suggested that Mae1p cannot act as a pyruvate-carboxylating, anaplerotic enzyme. In this study, relocation of malic enzyme to the cytosol and creation of thermodynamically favorable conditions for pyruvate carboxylation by metabolic engineering, process design, and adaptive evolution, enabled malic enzyme to act as the sole anaplerotic enzyme in S. cerevisiae. The Escherichia coli NADH-dependent sfcA malic enzyme was expressed in a Pyc(-) S. cerevisiae background. When PDC2, a transcriptional regulator of pyruvate decarboxylase genes, was deleted to increase intracellular pyruvate levels and cells were grown under a CO(2) atmosphere to favor carboxylation, adaptive evolution yielded a strain that grew on glucose (specific growth rate, 0.06 +/- 0.01 h(-1)). Growth of the evolved strain was enabled by a single point mutation (Asp336Gly) that switched the cofactor preference of E. coli malic enzyme from NADH to NADPH. Consistently, cytosolic relocalization of the native Mae1p, which can use both NADH and NADPH, in a pyc1,2 Delta pdc2 Delta strain grown under a CO(2) atmosphere, also enabled slow-growth on glucose. Although growth rates of these strains are still low, the higher ATP efficiency of carboxylation via malic enzyme, compared to the pyruvate carboxylase pathway, may contribute to metabolic engineering of S. cerevisiae for anaerobic, high-yield C(4)-dicarboxylic acid production.", + "extra": "Web of Science ID: WOS:000286597100004", "issue": "3", + "libraryCatalog": "Thomson Reuters Web of Knowledge", "pages": "732-738", - "publicationTitle": "Applied and Environmental Microbiology", + "publicationTitle": "APPLIED AND ENVIRONMENTAL MICROBIOLOGY", "volume": "77", "attachments": [], "tags": [], @@ -327,7 +551,7 @@ var testCases = [ }, { "itemType": "journalArticle", - "title": "Phosphoenolpyruvate Carboxykinase as the Sole Anaplerotic Enzyme in Saccharomyces cerevisiae", + "title": "Phosphoenolpyruvate Carboxykinase as the Sole Anaplerotic Enzyme in Saccharomyces cerevisiae", "creators": [ { "firstName": "Rintze M.", @@ -355,14 +579,14 @@ var testCases = [ "creatorType": "author" } ], - "date": "AUG 2010", + "date": "AUG 2010", "DOI": "10.1128/AEM.01077-10", "ISSN": "0099-2240", - "abstractNote": "Pyruvate carboxylase is the sole anaplerotic enzyme in glucose-grown cultures of wild-type Saccharomyces cerevisiae. Pyruvate carboxylase-negative (Pyc(-)) S. cerevisiae strains cannot grow on glucose unless media are supplemented with C(4) compounds, such as aspartic acid. In several succinate-producing prokaryotes, phosphoenolpyruvate carboxykinase (PEPCK) fulfills this anaplerotic role. However, the S. cerevisiae PEPCK encoded by PCK1 is repressed by glucose and is considered to have a purely decarboxylating and gluconeogenic function. This study investigates whether and under which conditions PEPCK can replace the anaplerotic function of pyruvate carboxylase in S. cerevisiae. Pyc(-) S. cerevisiae strains constitutively overexpressing the PEPCK either from S. cerevisiae or from Actinobacillus succinogenes did not grow on glucose as the sole carbon source. However, evolutionary engineering yielded mutants able to grow on glucose as the sole carbon source at a maximum specific growth rate of ca. 0.14 h(-1), one-half that of the (pyruvate carboxylase-positive) reference strain grown under the same conditions. Growth was dependent on high carbon dioxide concentrations, indicating that the reaction catalyzed by PEPCK operates near thermodynamic equilibrium. Analysis and reverse engineering of two independently evolved strains showed that single point mutations in pyruvate kinase, which competes with PEPCK for phosphoenolpyruvate, were sufficient to enable the use of PEPCK as the sole anaplerotic enzyme. The PEPCK reaction produces one ATP per carboxylation event, whereas the original route through pyruvate kinase and pyruvate carboxylase is ATP neutral. This increased ATP yield may prove crucial for engineering of efficient and low-cost anaerobic production of C(4) dicarboxylic acids in S. cerevisiae.", - "extra": "WOS:000280633400006", + "abstractNote": "Pyruvate carboxylase is the sole anaplerotic enzyme in glucose-grown cultures of wild-type Saccharomyces cerevisiae. Pyruvate carboxylase-negative (Pyc(-)) S. cerevisiae strains cannot grow on glucose unless media are supplemented with C(4) compounds, such as aspartic acid. In several succinate-producing prokaryotes, phosphoenolpyruvate carboxykinase (PEPCK) fulfills this anaplerotic role. However, the S. cerevisiae PEPCK encoded by PCK1 is repressed by glucose and is considered to have a purely decarboxylating and gluconeogenic function. This study investigates whether and under which conditions PEPCK can replace the anaplerotic function of pyruvate carboxylase in S. cerevisiae. Pyc(-) S. cerevisiae strains constitutively overexpressing the PEPCK either from S. cerevisiae or from Actinobacillus succinogenes did not grow on glucose as the sole carbon source. However, evolutionary engineering yielded mutants able to grow on glucose as the sole carbon source at a maximum specific growth rate of ca. 0.14 h(-1), one-half that of the (pyruvate carboxylase-positive) reference strain grown under the same conditions. Growth was dependent on high carbon dioxide concentrations, indicating that the reaction catalyzed by PEPCK operates near thermodynamic equilibrium. Analysis and reverse engineering of two independently evolved strains showed that single point mutations in pyruvate kinase, which competes with PEPCK for phosphoenolpyruvate, were sufficient to enable the use of PEPCK as the sole anaplerotic enzyme. The PEPCK reaction produces one ATP per carboxylation event, whereas the original route through pyruvate kinase and pyruvate carboxylase is ATP neutral. This increased ATP yield may prove crucial for engineering of efficient and low-cost anaerobic production of C(4) dicarboxylic acids in S. cerevisiae.", + "extra": "Web of Science ID: WOS:000280633400006", "issue": "16", "pages": "5383-5389", - "publicationTitle": "Applied and Environmental Microbiology", + "publicationTitle": "APPLIED AND ENVIRONMENTAL MICROBIOLOGY", "volume": "76", "attachments": [], "tags": [], @@ -371,7 +595,7 @@ var testCases = [ }, { "itemType": "journalArticle", - "title": "Key Process Conditions for Production of C(4) Dicarboxylic Acids in Bioreactor Batch Cultures of an Engineered Saccharomyces cerevisiae Strain", + "title": "Key Process Conditions for Production of C(4) Dicarboxylic Acids in Bioreactor Batch Cultures of an Engineered Saccharomyces cerevisiae Strain", "creators": [ { "firstName": "Rintze M.", @@ -399,14 +623,14 @@ var testCases = [ "creatorType": "author" } ], - "date": "FEB 2010", + "date": "FEB 2010", "DOI": "10.1128/AEM.02396-09", "ISSN": "0099-2240", - "abstractNote": "A recent effort to improve malic acid production by Saccharomyces cerevisiae by means of metabolic engineering resulted in a strain that produced up to 59 g liter(-1) of malate at a yield of 0.42 mol (mol glucose)(-1) in calcium carbonate-buffered shake flask cultures. With shake flasks, process parameters that are important for scaling up this process cannot be controlled independently. In this study, growth and product formation by the engineered strain were studied in bioreactors in order to separately analyze the effects of pH, calcium, and carbon dioxide and oxygen availability. A near-neutral pH, which in shake flasks was achieved by adding CaCO(3), was required for efficient C(4) dicarboxylic acid production. Increased calcium concentrations, a side effect of CaCO(3) dissolution, had a small positive effect on malate formation. Carbon dioxide enrichment of the sparging gas (up to 15% [vol/vol]) improved production of both malate and succinate. At higher concentrations, succinate titers further increased, reaching 0.29 mol (mol glucose)(-1), whereas malate formation strongly decreased. Although fully aerobic conditions could be achieved, it was found that moderate oxygen limitation benefitted malate production. In conclusion, malic acid production with the engineered S. cerevisiae strain could be successfully transferred from shake flasks to 1-liter batch bioreactors by simultaneous optimization of four process parameters (pH and concentrations of CO(2), calcium, and O(2)). Under optimized conditions, a malate yield of 0.48 +/- 0.01 mol (mol glucose)(-1) was obtained in bioreactors, a 19% increase over yields in shake flask experiments.", - "extra": "WOS:000274017400015", + "abstractNote": "A recent effort to improve malic acid production by Saccharomyces cerevisiae by means of metabolic engineering resulted in a strain that produced up to 59 g liter(-1) of malate at a yield of 0.42 mol (mol glucose)(-1) in calcium carbonate-buffered shake flask cultures. With shake flasks, process parameters that are important for scaling up this process cannot be controlled independently. In this study, growth and product formation by the engineered strain were studied in bioreactors in order to separately analyze the effects of pH, calcium, and carbon dioxide and oxygen availability. A near-neutral pH, which in shake flasks was achieved by adding CaCO(3), was required for efficient C(4) dicarboxylic acid production. Increased calcium concentrations, a side effect of CaCO(3) dissolution, had a small positive effect on malate formation. Carbon dioxide enrichment of the sparging gas (up to 15% [vol/vol]) improved production of both malate and succinate. At higher concentrations, succinate titers further increased, reaching 0.29 mol (mol glucose)(-1), whereas malate formation strongly decreased. Although fully aerobic conditions could be achieved, it was found that moderate oxygen limitation benefitted malate production. In conclusion, malic acid production with the engineered S. cerevisiae strain could be successfully transferred from shake flasks to 1-liter batch bioreactors by simultaneous optimization of four process parameters (pH and concentrations of CO(2), calcium, and O(2)). Under optimized conditions, a malate yield of 0.48 +/- 0.01 mol (mol glucose)(-1) was obtained in bioreactors, a 19% increase over yields in shake flask experiments.", + "extra": "Web of Science ID: WOS:000274017400015", "issue": "3", "pages": "744-750", - "publicationTitle": "Applied and Environmental Microbiology", + "publicationTitle": "APPLIED AND ENVIRONMENTAL MICROBIOLOGY", "volume": "76", "attachments": [], "tags": [], @@ -415,7 +639,7 @@ var testCases = [ }, { "itemType": "journalArticle", - "title": "Metabolic engineering of Saccharomyces cerevisiae for production of carboxylic acids: current status and challenges", + "title": "Metabolic engineering of Saccharomyces cerevisiae for production of carboxylic acids: current status and challenges", "creators": [ { "firstName": "Derek A.", @@ -438,14 +662,14 @@ var testCases = [ "creatorType": "author" } ], - "date": "DEC 2009", + "date": "DEC 2009", "DOI": "10.1111/j.1567-1364.2009.00537.x", "ISSN": "1567-1356", - "abstractNote": "To meet the demands of future generations for chemicals and energy and to reduce the environmental footprint of the chemical industry, alternatives for petrochemistry are required. Microbial conversion of renewable feedstocks has a huge potential for cleaner, sustainable industrial production of fuels and chemicals. Microbial production of organic acids is a promising approach for production of chemical building blocks that can replace their petrochemically derived equivalents. Although Saccharomyces cerevisiae does not naturally produce organic acids in large quantities, its robustness, pH tolerance, simple nutrient requirements and long history as an industrial workhorse make it an excellent candidate biocatalyst for such processes. Genetic engineering, along with evolution and selection, has been successfully used to divert carbon from ethanol, the natural endproduct of S. cerevisiae, to pyruvate. Further engineering, which included expression of heterologous enzymes and transporters, yielded strains capable of producing lactate and malate from pyruvate. Besides these metabolic engineering strategies, this review discusses the impact of transport and energetics as well as the tolerance towards these organic acids. In addition to recent progress in engineering S. cerevisiae for organic acid production, the key limitations and challenges are discussed in the context of sustainable industrial production of organic acids from renewable feedstocks.", - "extra": "WOS:000271264400001", + "abstractNote": "To meet the demands of future generations for chemicals and energy and to reduce the environmental footprint of the chemical industry, alternatives for petrochemistry are required. Microbial conversion of renewable feedstocks has a huge potential for cleaner, sustainable industrial production of fuels and chemicals. Microbial production of organic acids is a promising approach for production of chemical building blocks that can replace their petrochemically derived equivalents. Although Saccharomyces cerevisiae does not naturally produce organic acids in large quantities, its robustness, pH tolerance, simple nutrient requirements and long history as an industrial workhorse make it an excellent candidate biocatalyst for such processes. Genetic engineering, along with evolution and selection, has been successfully used to divert carbon from ethanol, the natural endproduct of S. cerevisiae, to pyruvate. Further engineering, which included expression of heterologous enzymes and transporters, yielded strains capable of producing lactate and malate from pyruvate. Besides these metabolic engineering strategies, this review discusses the impact of transport and energetics as well as the tolerance towards these organic acids. In addition to recent progress in engineering S. cerevisiae for organic acid production, the key limitations and challenges are discussed in the context of sustainable industrial production of organic acids from renewable feedstocks.", + "extra": "Web of Science ID: WOS:000271264400001", "issue": "8", "pages": "1123-1136", - "publicationTitle": "Fems Yeast Research", + "publicationTitle": "FEMS YEAST RESEARCH", "volume": "9", "attachments": [], "tags": [], @@ -454,7 +678,7 @@ var testCases = [ }, { "itemType": "journalArticle", - "title": "Malic acid production by Saccharomyces cerevisiae: Engineering of pyruvate carboxylation, oxaloacetate reduction, and malate export", + "title": "Malic acid production by Saccharomyces cerevisiae: Engineering of pyruvate carboxylation, oxaloacetate reduction, and malate export", "creators": [ { "firstName": "Rintze M.", @@ -507,14 +731,14 @@ var testCases = [ "creatorType": "author" } ], - "date": "MAY 2008", + "date": "MAY 2008", "DOI": "10.1128/AEM.02591-07", "ISSN": "0099-2240", - "abstractNote": "Malic acid is a potential biomass-derivable \"building block\" for chemical synthesis. Since wild-type Saccharomyces cerevisiae strains produce only low levels of malate, metabolic engineering is required to achieve efficient malate production with this yeast. A promising pathway for malate production from glucose proceeds via carboxylation of pyruvate, followed by reduction of oxaloacetate to malate. This redox- and ATP-neutral, CO2-fixing pathway has a theoretical maximum yield of 2 mol malate (mol glucose)(-1). A previously engineered glucose-tolerant, C-2-independent pyruvate decarboxylase-negative S. cerevisiae strain was used as the platform to evaluate the impact of individual and combined introduction of three genetic modifications: (i) overexpression of the native pyruvate carboxylase encoded by PYC2, (ii) high-level expression of an allele of the MDH3 gene, of which the encoded malate dehydrogenase was retargeted to the cytosol by deletion of the C-terminal peroxisomal targeting sequence, and (iii) functional expression of the Schizosaccharomyces pombe malate transporter gene SpMAE1. While single or double modifications improved malate production, the highest malate yields and titers were obtained with the simultaneous introduction of all three modifications. In glucose-grown batch cultures, the resulting engineered strain produced malate at titers of up to 59 g liter(-1) at a malate yield of 0.42 mol (mol glucose)(-1). Metabolic flux analysis showed that metabolite labeling patterns observed upon nuclear magnetic resonance analyses of cultures grown on C-13-labeled glucose were consistent with the envisaged nonoxidative, fermentative pathway for malate production. The engineered strains still produced substantial amounts of pyruvate, indicating that the pathway efficiency can be further improved.", - "extra": "WOS:000255567900024", + "abstractNote": "Malic acid is a potential biomass-derivable \"building block\" for chemical synthesis. Since wild-type Saccharomyces cerevisiae strains produce only low levels of malate, metabolic engineering is required to achieve efficient malate production with this yeast. A promising pathway for malate production from glucose proceeds via carboxylation of pyruvate, followed by reduction of oxaloacetate to malate. This redox- and ATP-neutral, CO2-fixing pathway has a theoretical maximum yield of 2 mol malate (mol glucose)(-1). A previously engineered glucose-tolerant, C-2-independent pyruvate decarboxylase-negative S. cerevisiae strain was used as the platform to evaluate the impact of individual and combined introduction of three genetic modifications: (i) overexpression of the native pyruvate carboxylase encoded by PYC2, (ii) high-level expression of an allele of the MDH3 gene, of which the encoded malate dehydrogenase was retargeted to the cytosol by deletion of the C-terminal peroxisomal targeting sequence, and (iii) functional expression of the Schizosaccharomyces pombe malate transporter gene SpMAE1. While single or double modifications improved malate production, the highest malate yields and titers were obtained with the simultaneous introduction of all three modifications. In glucose-grown batch cultures, the resulting engineered strain produced malate at titers of up to 59 g liter(-1) at a malate yield of 0.42 mol (mol glucose)(-1). Metabolic flux analysis showed that metabolite labeling patterns observed upon nuclear magnetic resonance analyses of cultures grown on C-13-labeled glucose were consistent with the envisaged nonoxidative, fermentative pathway for malate production. The engineered strains still produced substantial amounts of pyruvate, indicating that the pathway efficiency can be further improved.", + "extra": "Web of Science ID: WOS:000255567900024", "issue": "9", "pages": "2766-2777", - "publicationTitle": "Applied and Environmental Microbiology", + "publicationTitle": "APPLIED AND ENVIRONMENTAL MICROBIOLOGY", "volume": "74", "attachments": [], "tags": [], @@ -564,29 +788,60 @@ var testCases = [ ], "date": "1998", "ISSN": "0161-3499", - "extra": "CABI:19982209000", + "extra": "Web of Science ID: CABI:19982209000", "issue": "2", "language": "English", + "libraryCatalog": "Thomson Reuters Web of Knowledge", "pages": "170", "publicationTitle": "Veterinary Surgery", "volume": "27", "attachments": [], "tags": [ - "Canidae", - "Canis", - "Chordata", - "Fissipeda", - "anaesthesia", - "animals", - "carnivores", - "dogs", - "eukaryotes", - "halothane", - "mammals", - "muscle relaxants", - "pharmacokinetics", - "small mammals", - "vertebrates" + { + "tag": "Canidae" + }, + { + "tag": "Canis" + }, + { + "tag": "Chordata" + }, + { + "tag": "Fissipeda" + }, + { + "tag": "anaesthesia" + }, + { + "tag": "animals" + }, + { + "tag": "carnivores" + }, + { + "tag": "dogs" + }, + { + "tag": "eukaryotes" + }, + { + "tag": "halothane" + }, + { + "tag": "mammals" + }, + { + "tag": "muscle relaxants" + }, + { + "tag": "pharmacokinetics" + }, + { + "tag": "small mammals" + }, + { + "tag": "vertebrates" + } ], "notes": [], "seeAlso": [] @@ -609,14 +864,13 @@ var testCases = [ ], "date": "SPR 2011", "ISSN": "0015-0630", - "extra": "WOS:000290115300030", + "extra": "Web of Science ID: WOS:000290115300030", "issue": "247", "journalAbbreviation": "Fiddlehead", "language": "English", + "libraryCatalog": "Thomson Reuters Web of Knowledge", "pages": "82-82", - "place": "Fredericton", - "publicationTitle": "Fiddlehead", - "publisher": "Univ New Brunswick", + "publicationTitle": "FIDDLEHEAD", "attachments": [], "tags": [], "notes": [], @@ -669,33 +923,57 @@ var testCases = [ } ], "date": "2011", - "DOI": "10.1146/annurev-publhealth-090810-182459", "ISBN": "978-0-8243-2732-3", - "ISSN": "0163-7525", - "abstractNote": "The high rate of premature births in the United States remains a public health concern. These infants experience substantial morbidity and mortality in the newborn period, which translate into significant medical costs. In early childhood, survivors are characterized by a variety of health problems, including motor delay and/or cerebral palsy, lower IQs, behavior problems, and respiratory illness, especially asthma. Many experience difficulty with school work, lower health-related quality of life, and family stress. Emerging information in adolescence and young adulthood paints a more optimistic picture, with persistence of many problems but with better adaptation and more positive expectations by the young adults. Few opportunities for prevention have been identified; therefore, public health approaches to prematurity include assurance of delivery in a facility capable of managing neonatal complications, quality improvement to minimize interinstitutional variations, early developmental support for such infants, and attention to related family health issues.", - "extra": "WOS:000290776200020", - "journalAbbreviation": "Annu. Rev. Public Health", + "abstractNote": "The high rate of premature births in the United States remains a public health concern. These infants experience substantial morbidity and mortality in the newborn period, which translate into significant medical costs. In early childhood, survivors are characterized by a variety of health problems, including motor delay and/or cerebral palsy, lower IQs, behavior problems, and respiratory illness, especially asthma. Many experience difficulty with school work, lower health-related quality of life, and family stress. Emerging information in adolescence and young adulthood paints a more optimistic picture, with persistence of many problems but with better adaptation and more positive expectations by the young adults. Few opportunities for prevention have been identified; therefore, public health approaches to prematurity include assurance of delivery in a facility capable of managing neonatal complications, quality improvement to minimize interinstitutional variations, early developmental support for such infants, and attention to related family health issues.", + "bookTitle": "ANNUAL REVIEW OF PUBLIC HEALTH, VOL 32", + "extra": "Web of Science ID: WOS:000290776200020", "language": "English", + "libraryCatalog": "Thomson Reuters Web of Knowledge", "pages": "367-379", "place": "Palo Alto", - "publicationTitle": "Annual Review of Public Health, Vol 32", "publisher": "Annual Reviews", "volume": "32", "attachments": [], "tags": [ - "age 8 years", - "bronchopulmonary dysplasia", - "childhood morbidity", - "children born", - "extreme prematurity", - "extremely preterm birth", - "infant mortality", - "learning-disabilities", - "low-birth-weight", - "neonatal intensive-care", - "prevention", - "quality-of-life", - "young-adults born" + { + "tag": "AGE 8 YEARS" + }, + { + "tag": "BRONCHOPULMONARY DYSPLASIA" + }, + { + "tag": "CHILDREN BORN" + }, + { + "tag": "EXTREME PREMATURITY" + }, + { + "tag": "EXTREMELY PRETERM BIRTH" + }, + { + "tag": "LEARNING-DISABILITIES" + }, + { + "tag": "LOW-BIRTH-WEIGHT" + }, + { + "tag": "NEONATAL INTENSIVE-CARE" + }, + { + "tag": "QUALITY-OF-LIFE" + }, + { + "tag": "YOUNG-ADULTS BORN" + }, + { + "tag": "childhood morbidity" + }, + { + "tag": "infant mortality" + }, + { + "tag": "prevention" + } ], "notes": [], "seeAlso": [] @@ -731,24 +1009,27 @@ var testCases = [ "creatorType": "inventor" } ], - "date": "JUN 28 2011", - "issueDate": "June 28, 2011", - "ISSN": "0098-1133", - "abstractNote": "An insertion device with an insertion axis includes an axial actuator with a first portion and a second portion. The first portion is moveable along the insertion axis relative to the second portion. The insertion device further includes a first tube coupled to the first portion of the axial actuator, and the first tube is movable along the insertion axis in response to movement of the first portion relative to the second portion. The device further includes a second tube having a radially biased distal end. The distal end is substantially contained within the first tube in a first state, and the second tube is rotatable with respect to the first tube. Also, the second tube is axially movable to a second state, and a portion of a distal end of the second tube is exposed from a distal end of the first tube in the second state.", + "issueDate": "JUN 28 2011", + "abstractNote": "An insertion device with an insertion axis includes an axial actuator with a first portion and a second portion. The first portion is moveable along the insertion axis relative to the second portion. The insertion device further includes a first tube coupled to the first portion of the axial actuator, and the first tube is movable along the insertion axis in response to movement of the first portion relative to the second portion. The device further includes a second tube having a radially biased distal end. The distal end is substantially contained within the first tube in a first state, and the second tube is rotatable with respect to the first tube. Also, the second tube is axially movable to a second state, and a portion of a distal end of the second tube is exposed from a distal end of the first tube in the second state.", "assignee": "Medtronic Inc", "country": "USA", - "extra": "BIOSIS:PREV201100469175", + "extra": "Web of Science ID: BIOSIS:PREV201100469175", "language": "English", "patentNumber": "US 07967789", - "place": "Indialantic, FL USA", - "priorityNumber": "604-16501", - "publicationTitle": "Official Gazette of the United States Patent and Trademark Office Patents", "attachments": [], "tags": [ - "Equipment Apparatus Devices and Instrumentation", - "Human Medicine (Medical Sciences)", - "indexing cell delivery catheter", - "medical supplies" + { + "tag": "Equipment Apparatus Devices and Instrumentation" + }, + { + "tag": "Human Medicine (Medical Sciences)" + }, + { + "tag": "indexing cell delivery catheter" + }, + { + "tag": "medical supplies" + } ], "notes": [], "seeAlso": [] @@ -771,8 +1052,8 @@ var testCases = [ ], "date": "2011", "ISBN": "978-90-8585933-8", - "extra": "CABI:20113178956", - "publicationTitle": "Ecological anthropology of households in East Madura, Indonesia", + "extra": "Web of Science ID: CABI:20113178956", + "libraryCatalog": "Thomson Reuters Web of Knowledge", "attachments": [], "tags": [], "notes": [], @@ -780,7 +1061,7 @@ var testCases = [ }, { "itemType": "journalArticle", - "title": "Production and characterization of polyclonal antibodies to hexanal-lysine adducts for use in an ELISA to monitor lipid oxidation in a meat model system.", + "title": "Production and characterization of polyclonal antibodies to hexanal-lysine adducts for use in an ELISA to monitor lipid oxidation in a meat model system.", "creators": [ { "firstName": "S. A.", @@ -788,9 +1069,9 @@ var testCases = [ "creatorType": "author" } ], - "date": ", thesis publ. 1997 1998", + "date": "1998, thesis publ. 1997", "ISSN": "0419-4217", - "extra": "FSTA:1998-09-Sn1570", + "extra": "Web of Science ID: FSTA:1998-09-Sn1570", "issue": "9", "publicationTitle": "Dissertation Abstracts International, B", "volume": "58", @@ -811,7 +1092,7 @@ var testCases = [ ], "date": "1988", "ISSN": "0419-4217", - "extra": "FSTA:1989-04-N-0004", + "extra": "Web of Science ID: FSTA:1989-04-N-0004", "issue": "4", "pages": "BRD", "publicationTitle": "Dissertation Abstracts International, B", @@ -833,7 +1114,7 @@ var testCases = [ ], "date": "1988", "ISSN": "0419-4217", - "extra": "CABI:19910448509", + "extra": "Web of Science ID: CABI:19910448509", "issue": "5", "pages": "1459", "publicationTitle": "Dissertation Abstracts International. B, Sciences and Engineering", @@ -866,10 +1147,11 @@ var testCases = [ ], "date": "JAN 2013", "DOI": "10.1007/s10750-012-1232-8", - "ISSN": "0018-8158", + "ISSN": "0018-8158, 1573-5117", "abstractNote": "Microbial processing of detritus is known to be important to benthic invertebrate nutrition, but the role of dissolved (DOC) versus particulate organic carbon (POC), and pathways by which those resources are obtained, are poorly understood. We used stable isotopes to determine the importance of DOC, POC, and CH4-derived carbon to benthic invertebrate consumers from arctic Alaskan Lakes. Intact sediment cores from Lake GTH 112 were enriched with C-13-labeled organic matter, including algal detritus, algal-derived DOC, methyl-labeled acetate, and carboxyl-labeled acetate, and incubated for 1 month with either caddisflies (Grensia praeterita ) or fingernail clams (Sphaerium nitidum), two invertebrate species that are important to fish nutrition. Both species used basal resources derived from POC and DOC. Results generally suggest greater reliance on POC. Differential assimilation from acetate treatments suggests Sphaerium assimilated CH4-derived carbon, which likely occurred through deposit-feeding. Grensia assimilated some microbially processed acetate, although its survivorship was poor in acetate treatments. Our data extend previous studies reporting use of CH4-derived carbon by Chironomidae and oligochaetes. Taken together, these results suggest that the use of CH4-derived carbon is common among deposit-feeding benthic invertebrates.", - "extra": "BCI:BCI201300112663", + "extra": "Web of Science ID: BCI:BCI201300112663", "issue": "1", + "libraryCatalog": "Thomson Reuters Web of Knowledge", "pages": "221-230", "publicationTitle": "Hydrobiologia", "volume": "700", @@ -889,19 +1171,19 @@ var testCases = [ "title": "Use of reagent for detecting syntaxin 12 protein autoantibodies in preparation of lung cancer screening kit", "creators": [ { - "firstName": "YANG", - "lastName": "Y", + "firstName": "Y.", + "lastName": "YANG", "creatorType": "inventor" }, { - "firstName": "LI", - "lastName": "W", + "firstName": "W.", + "lastName": "LI", "creatorType": "inventor" } ], "abstractNote": "NOVELTY - Use of reagent for detecting syntaxin 12 (STX12) protein autoantibodies, is claimed in preparation of a lung cancer screening kit. USE - The reagent for detecting STX12 protein autoantibodies is useful in preparation of a lung cancer screening kit (claimed). ADVANTAGE - The reagent realizes effective screening of lung cancer and detects that the autoantibody level of the STX12 protein in the serum of lung cancer patients is significantly lower than that of healthy patients.", - "assignee": "UNIV SICHUAN WEST CHINA HOSPITAL", - "extra": "DIIDW:202018799C", + "assignee": "Univ Sichuan West China Hospital", + "extra": "Web of Science ID: DIIDW:202018799C", "patentNumber": "CN110836969-A", "attachments": [], "tags": [], @@ -927,16 +1209,18 @@ var testCases = [ "date": "2005", "DOI": "10.1109/ICACT.2005.245926", "abstractNote": "Increased speeds of PCs and networks have made video conferencing systems possible in Internet. The proposed conference control protocol suits small scale video conferencing systems which employ full mesh conferencing architecture and loosely coupled conferencing mode. The protocol can ensure the number of conference member is less than the maximum value. Instant message services are used to do member authentication and notification. The protocol is verified in 32 concurrent conferencing scenarios and implemented in DigiParty which is a small scale video conferencing add-in application for MSN Messenger.", - "extra": "WOS:000230445900101", + "conferenceName": "7th International Conference on Advanced Communication Technology", + "extra": "Web of Science ID: WOS:000230445900101", "language": "English", + "libraryCatalog": "Clarivate Analytics Web of Science", "pages": "532-537", "place": "New York", - "proceedingsTitle": "7th International Conference on Advanced Communication Technology, Vols 1 and 2, Proceedings", - "publisher": "Ieee", + "proceedingsTitle": "7TH INTERNATIONAL CONFERENCE ON ADVANCED COMMUNICATION TECHNOLOGY, VOLS 1 AND 2, PROCEEDINGS", + "publisher": "IEEE", "attachments": [], "tags": [ { - "tag": "conference control protocol" + "tag": "conference control protocol" }, { "tag": "full mesh" diff --git a/Wikidata.js b/Wikidata.js index 993d4bee6c6..67a1afdcdb8 100644 --- a/Wikidata.js +++ b/Wikidata.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2021-07-29 16:07:10" + "lastUpdated": "2023-08-19 09:56:11" } /* @@ -72,6 +72,7 @@ var typeMapping = { Q10870555: "report", Q820655: "statute", Q1266946: "thesis", + Q187685: "thesis", // doctoral thesis Q15416: "tvBroadcast", Q30070675: "videoRecording", Q36774: "webpage", @@ -209,7 +210,7 @@ function scrape(doc, url) { // Z.debug("Internal look up resource: " + resource); value = ZU.xpathText(xml, '//rdf:Description[@rdf:about="' + resource + '"]/rdfs:label[contains(@xml:lang, "en")][1]', namespaces) || ZU.xpathText(xml, '//rdf:Description[@rdf:about="' + resource + '"]/rdfs:label[1]', namespaces); - + if (!value) { continue; } // Z.debug(value); } if (zprop == "creator") { @@ -830,6 +831,81 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://www.wikidata.org/wiki/Q30834230", + "items": [ + { + "itemType": "journalArticle", + "creators": [ + { + "firstName": "Nicola", + "lastName": "Francesca", + "creatorType": "author" + }, + { + "firstName": "Cláudia", + "lastName": "Carvalho", + "creatorType": "author" + }, + { + "firstName": "Ciro", + "lastName": "Sannino", + "creatorType": "author" + }, + { + "firstName": "Marco Alexandre", + "lastName": "Guerreiro", + "creatorType": "author" + }, + { + "firstName": "Pedro", + "lastName": "Almeida", + "creatorType": "author" + }, + { + "firstName": "Luca", + "lastName": "Settanni", + "creatorType": "author" + }, + { + "firstName": "José Paulo", + "lastName": "Sampaio", + "creatorType": "author" + }, + { + "firstName": "Bruno", + "lastName": "Massa", + "creatorType": "author" + }, + { + "firstName": "Giancarlo", + "lastName": "Moschetti", + "creatorType": "author" + } + ], + "tags": [ + { + "tag": "species nova", + "type": 1 + }, + { + "tag": "species nova", + "type": 1 + } + ], + "extra": "QID: Q30834230\nPMID: 24981278", + "date": "2014-07-28T00:00:00Z", + "issue": "6", + "publicationTitle": "FEMS Yeast Research, FEMS Yeast Research", + "DOI": "10.1111/1567-1364.12179", + "volume": "14", + "title": "Yeasts vectored by migratory birds collected in the Mediterranean island of Ustica and description of Phaffomyces usticensis f.a. sp. nov., a new species related to the cactus ecoclade", + "pages": "910-921", + "libraryCatalog": "Wikidata" + } + ] } ] /** END TEST CASES **/ diff --git a/Wikipedia Citation Templates.js b/Wikipedia Citation Templates.js index 911ee3c12ca..8387d110ec5 100644 --- a/Wikipedia Citation Templates.js +++ b/Wikipedia Citation Templates.js @@ -11,7 +11,7 @@ }, "inRepository": true, "translatorType": 2, - "lastUpdated": "2023-01-03 10:30:59" + "lastUpdated": "2023-11-05 21:29:00" } /* @@ -319,7 +319,7 @@ function doExport() { // Don't include access date for journals with no URL if (item.accessDate && !(item.itemType == 'journalArticle' && !item.url)) { - properties.accessdate = formatDate(item.accessDate); + properties["access-date"] = formatDate(item.accessDate); } if (item.date) { diff --git a/ZOBODAT.js b/ZOBODAT.js new file mode 100644 index 00000000000..3c5159f6c86 --- /dev/null +++ b/ZOBODAT.js @@ -0,0 +1,341 @@ +{ + "translatorID": "e94ffd1c-0ff8-4fbc-8b2a-8391ab5a7288", + "label": "ZOBODAT", + "creator": "Lars Willighagen", + "target": "^https?://(www\\.)?zobodat\\.at/", + "minVersion": "5.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2023-10-17 11:17:19" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2023 Lars Willighagen + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see <http://www.gnu.org/licenses/>. + + ***** END LICENSE BLOCK ***** +*/ + + +function detectWeb(doc, url) { + if (url.includes('/publikation_articles.php')) { + return 'journalArticle'; + } + else if ((url.includes('/publikation_series.php') || url.includes('/publikation_volumes.php')) && getSearchResults(doc)) { + return 'multiple'; + } + return false; +} + +function getSearchResults(doc) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('.result'); + for (var i = 0; i < rows.length; i++) { + var href = rows[i].querySelector(['a[href^="publikation_articles.php"]']); + var title = ZU.trimInternal(text(rows[i], '.title')); + if (!href || !title) { + continue; + } + else { + found = true; + items[href.href] = title; + } + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) === 'multiple') { + let items = await Zotero.selectItems(getSearchResults(doc)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); + } + } + else { + await scrape(doc, url); + } +} + +async function scrape(doc, url = doc.location.href) { + var item = new Zotero.Item('journalArticle'); + item.url = url; + + var reference = doc.querySelector('#publikation_articles h1 + div'); + + var referenceParts = ZU.trimInternal(reference.textContent).split(' – '); + var authorshipParts = referenceParts[0].match(/^(.+) \((\d+)\): (.+)$/); + var locatorParts = referenceParts[referenceParts.length - 1].match(/(.+): (.+ - .+)\.$/); + + // Authors + var authors = authorshipParts[1].split(', '); + for (var i = 0; i < authors.length; i++) { + item.creators.push(ZU.cleanAuthor(authors[i], 'author')); + } + + // Publication date + item.date = authorshipParts[2]; + + // Title (+ PDF) + var pdfLink = reference.querySelector('a[href^="pdf"]'); + if (pdfLink) { + item.title = pdfLink.textContent; + item.attachments.push({ title: 'Full Text PDF', mimeType: 'application/pdf', url: pdfLink.href }); + } + else { + item.title = authorshipParts[3]; + } + + // Journal + item.publicationTitle = text(reference, 'a[href^="publikation_series"]') || referenceParts[1]; + + // Locator + var volume = text(reference, 'a[href^="publikation_volumes"]') || locatorParts[1]; + var volumeParts = volume.match(/^(\d+)_(\d+)$/); + if (volumeParts) { + item.volume = cleanNumber(volumeParts[1]); + item.issue = cleanNumber(volumeParts[2]); + } + else { + item.volume = cleanNumber(volume); + } + + item.pages = locatorParts[2].split(' - ').map(cleanNumber).join('-'); + + item.complete(); +} + +function cleanNumber(number) { + return number.replace(/^0+([1-9]\d*)/g, '$1'); +} + +/** BEGIN TEST CASES **/ +var testCases = [ + { + "type": "web", + "url": "https://www.zobodat.at/publikation_articles.php?id=275236", + "items": [ + { + "itemType": "journalArticle", + "title": "Käferlarven und Käferpuppen aus Maulwurfsnestern", + "creators": [ + { + "lastName": "Beier", + "firstName": "Max Walter Peter", + "creatorType": "author" + }, + { + "lastName": "Strouhal", + "firstName": "Hans", + "creatorType": "author" + } + ], + "date": "1928", + "libraryCatalog": "ZOBODAT", + "pages": "1-34", + "publicationTitle": "Zeitschrift für wissenschaftliche Insektenbiologie", + "url": "https://www.zobodat.at/publikation_articles.php?id=275236", + "volume": "23", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.zobodat.at/publikation_articles.php?id=10021894", + "items": [ + { + "itemType": "journalArticle", + "title": "A provisional multispecies toxicity test using indigenous organisms", + "creators": [ + { + "firstName": "John", + "lastName": "Cairns", + "creatorType": "author" + }, + { + "firstName": "James R.", + "lastName": "Pratt", + "creatorType": "author" + }, + { + "firstName": "B. R.", + "lastName": "Niederlehner", + "creatorType": "author" + } + ], + "date": "1985", + "libraryCatalog": "ZOBODAT", + "pages": "316-319", + "publicationTitle": "J. Test. Eval.", + "url": "https://www.zobodat.at/publikation_articles.php?id=10021894", + "volume": "13", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.zobodat.at/publikation_volumes.php?id=48239", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.zobodat.at/publikation_series.php?id=20840", + "detectedItemType": false, + "items": [] + }, + { + "type": "web", + "url": "https://www.zobodat.at/publikation_series.php?q=&as_l%5B0%5D%5Bi%5D=surname&as_l%5B0%5D%5Bqt%5D=contains&as_l%5B0%5D%5Bv%5D=Achterberg&as_l%5B1%5D%5Bi%5D=", + "items": "multiple" + }, + { + "type": "web", + "url": "https://www.zobodat.at/personen.php?id=5312", + "detectedItemType": false, + "items": [] + }, + { + "type": "web", + "url": "https://www.zobodat.at/publikation_articles.php?id=256029", + "items": [ + { + "itemType": "journalArticle", + "title": "A faunistic study of Braconidae (Hymenoptera: Ichneumonoidea)\nfrom southern Iran", + "creators": [ + { + "lastName": "Samin", + "firstName": "Najmeh", + "creatorType": "author" + }, + { + "lastName": "Achterberg", + "firstName": "Cees van (auch Cornelis)", + "creatorType": "author" + }, + { + "lastName": "Ghahari", + "firstName": "Hassan", + "creatorType": "author" + } + ], + "date": "2015", + "issue": "2", + "libraryCatalog": "ZOBODAT", + "pages": "1801-1809", + "publicationTitle": "Linzer biologische Beiträge", + "shortTitle": "A faunistic study of Braconidae (Hymenoptera", + "url": "https://www.zobodat.at/publikation_articles.php?id=256029", + "volume": "47", + "attachments": [ + { + "title": "Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.zobodat.at/publikation_articles.php?id=10027526", + "items": [ + { + "itemType": "journalArticle", + "title": "The valid name for the genus *Loxocephalus* #Foerster#, 1862 (Insecta, Hymenoptera: Braconidae), preoccupied by *Loxocephalus* #Eberhard#, 1862 (Protozoa: Ciliophora)", + "creators": [ + { + "firstName": "Wilhelm", + "lastName": "Foissner", + "creatorType": "author" + }, + { + "firstName": "Cees van (auch Cornelis)", + "lastName": "Achterberg", + "creatorType": "author" + } + ], + "date": "1997", + "libraryCatalog": "ZOBODAT", + "pages": "31-32", + "publicationTitle": "Zooel.Meded.Leiden", + "shortTitle": "The valid name for the genus *Loxocephalus* #Foerster#, 1862 (Insecta, Hymenoptera", + "url": "https://www.zobodat.at/publikation_articles.php?id=10027526", + "volume": "71", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.zobodat.at/publikation_articles.php?id=521285", + "items": [ + { + "itemType": "journalArticle", + "title": "Kleintierhabitate", + "creators": [ + { + "firstName": "Daniela", + "lastName": "Hofinger", + "creatorType": "author" + }, + { + "firstName": "Harald", + "lastName": "Kutzenberger", + "creatorType": "author" + } + ], + "date": "2023", + "issue": "1", + "libraryCatalog": "ZOBODAT", + "pages": "11-14", + "publicationTitle": "ÖKO.L Zeitschrift für Ökologie, Natur- und Umweltschutz", + "url": "https://www.zobodat.at/publikation_articles.php?id=521285", + "volume": "2023", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + } +] +/** END TEST CASES **/ diff --git a/Zenodo.js b/Zenodo.js index d9026151222..5993c47a293 100644 --- a/Zenodo.js +++ b/Zenodo.js @@ -1,21 +1,21 @@ { "translatorID": "a714cb93-6595-482f-b371-a4ca0be14449", "label": "Zenodo", - "creator": "Philipp Zumstein, Sebastian Karcher", + "creator": "Philipp Zumstein, Sebastian Karcher and contributors", "target": "^https?://zenodo\\.org", - "minVersion": "3.0", + "minVersion": "6.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2023-04-19 20:55:45" + "lastUpdated": "2023-12-08 09:09:32" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2016, 2017 Philipp Zumstein & Sebastian Karcher + Copyright © 2016-2023 Philipp Zumstein, Sebastian Karcher and contributors This file is part of Zotero. @@ -40,11 +40,10 @@ const datasetType = ZU.fieldIsValidForType('title', 'dataset') : 'document'; function detectWeb(doc, url) { - if (url.includes('/record/')) { - var collections = ZU.xpath(doc, '//span[@class="pull-right"]/span[contains(@class, "label-default")]'); + if (url.includes('/records/')) { + var collections = doc.querySelectorAll('span[aria-label="Resource type"]'); for (var i = 0; i < collections.length; i++) { - var type = collections[i].textContent.toLowerCase(); - //Z.debug(type) + var type = collections[i].textContent.toLowerCase().trim(); switch (type) { case "software": return "computerProgram"; @@ -70,8 +69,9 @@ function detectWeb(doc, url) { case "report": case "working paper": case "project deliverables": - case "preprint": return "report"; + case "preprint": + return "preprint"; case "thesis": return "thesis"; case "dataset": @@ -92,7 +92,8 @@ function detectWeb(doc, url) { function getSearchResults(doc, checkOnly) { var items = {}; var found = false; - var rows = ZU.xpath(doc, '//invenio-search-results//h4/a'); + // this section is not rendered in the 6.0 Scaffold browser, OK in v7 + var rows = doc.querySelectorAll('section[aria-label="Search results"] h2 a'); for (var i = 0; i < rows.length; i++) { var href = rows[i].href; var title = ZU.trimInternal(rows[i].textContent); @@ -105,7 +106,7 @@ function getSearchResults(doc, checkOnly) { } -function doWeb(doc, url) { +async function doWeb(doc, url) { if (detectWeb(doc, url) == "multiple") { Zotero.selectItems(getSearchResults(doc, false), function (items) { if (!items) { @@ -119,110 +120,104 @@ function doWeb(doc, url) { }); } else { - scrape(doc, url); + await scrape(doc, url); } } -function scrape(doc, url) { +async function scrape(doc, url) { var abstract = ZU.xpathText(doc, '//meta[@name="description"]/@content'); var doi = ZU.xpathText(doc, '//meta[@name="citation_doi"]/@content'); var pdfURL = ZU.xpathText(doc, '//meta[@name="citation_pdf_url"]/@content'); var tags = ZU.xpath(doc, '//meta[@name="citation_keywords"]'); var cslURL = url.replace(/#.+/, "").replace(/\?.+/, "").replace(/\/export\/.+/, "") + "/export/csl"; - // Z.debug(cslURL) // use CSL JSON translator - ZU.processDocuments(cslURL, function (newDoc) { - var text = ZU.xpathText(newDoc, '//h3/following-sibling::pre'); - // Z. debug(text); - text = text.replace(/publisher_place/, "publisher-place"); - text = text.replace(/container_title/, "container-title"); + var text = await requestText(cslURL); + text = text.replace(/publisher_place/, "publisher-place"); + text = text.replace(/container_title/, "container-title"); + + var trans = Zotero.loadTranslator('import'); + trans.setTranslator('bc03b4fe-436d-4a1f-ba59-de4d2d7a63f7'); + trans.setString(text); + trans.setHandler("itemDone", function (obj, item) { + item.itemType = detectWeb(doc, url); + // The "note" field of CSL maps to Extra. Put it in a note instead + if (item.extra) { + item.notes.push({ note: item.extra }); + item.extra = ""; + } + if (!ZU.fieldIsValidForType('DOI', item.itemType) && doi) { + item.extra = "DOI: " + doi; + } - var trans = Zotero.loadTranslator('import'); - // CSL JSON - trans.setTranslator('bc03b4fe-436d-4a1f-ba59-de4d2d7a63f7'); - trans.setString(text); - trans.setHandler("itemDone", function (obj, item) { - item.itemType = detectWeb(doc, url); - // The "note" field of CSL maps to Extra. Put it in a note instead + // workaround for pre 6.0.24 versions that don't have proper item type for data + // if (schemaType && schemaType.includes("Dataset")) { + if (datasetType != "dataset" && ZU.xpathText(doc, '//span[@class="pull-right"]/span[contains(@class, "label-default") and contains(., "Dataset")]')) { if (item.extra) { - item.notes.push({ note: item.extra }); - item.extra = ""; + item.extra += "\nType: dataset"; } - if (!ZU.fieldIsValidForType('DOI', item.itemType) && doi) { - item.extra = "DOI: " + doi; + else { + item.extra = "Type: dataset"; } + } - // workaround for pre 6.0.24 versions that don't have proper item type for data - // if (schemaType && schemaType.includes("Dataset")) { - if (datasetType != "dataset" && ZU.xpathText(doc, '//span[@class="pull-right"]/span[contains(@class, "label-default") and contains(., "Dataset")]')) { - if (item.extra) { - item.extra += "\nType: dataset"; + //get PDF attachment, otherwise just snapshot. + if (pdfURL) { + item.attachments.push({ url: pdfURL, title: "Zenodo Full Text PDF", mimeType: "application/pdf" }); + } + else { + item.attachments.push({ url: url, title: "Zenodo Snapshot", mimeType: "text/html" }); + } + for (let i = 0; i < tags.length; i++) { + item.tags.push(tags[i].content); + } + + //something is odd with zenodo's author parsing to CSL on some pages; fix it + //e.g. https://zenodo.org/record/569323 + for (let i = 0; i < item.creators.length; i++) { + let creator = item.creators[i]; + if (!creator.firstName || !creator.firstName.length) { + if (creator.lastName.includes(",")) { + creator.firstName = creator.lastName.replace(/.+?,\s*/, ""); + creator.lastName = creator.lastName.replace(/,.+/, ""); } else { - item.extra = "Type: dataset"; + item.creators[i] = ZU.cleanAuthor(creator.lastName, + creator.creatorType, false); } } + delete item.creators[i].creatorTypeID; + } - //get PDF attachment, otherwise just snapshot. - if (pdfURL) { - item.attachments.push({ url: pdfURL, title: "Zenodo Full Text PDF", mimeType: "application/pdf" }); - } - else { - item.attachments.push({ url: url, title: "Zenodo Snapshot", mimeType: "text/html" }); - } - for (let i = 0; i < tags.length; i++) { - item.tags.push(tags[i].content); - } - - //something is odd with zenodo's author parsing to CSL on some pages; fix it - //e.g. https://zenodo.org/record/569323 - for (let i = 0; i < item.creators.length; i++) { - let creator = item.creators[i]; - if (!creator.firstName || !creator.firstName.length) { - if (creator.lastName.includes(",")) { - creator.firstName = creator.lastName.replace(/.+?,\s*/, ""); - creator.lastName = creator.lastName.replace(/,.+/, ""); - } - else { - item.creators[i] = ZU.cleanAuthor(creator.lastName, - creator.creatorType, false); - } - } - delete item.creators[i].creatorTypeID; - } - - //Don't use Zenodo as university for theses -- but use as archive - if (item.itemType == "thesis" && item.publisher == "Zenodo") { - item.publisher = ""; - item.archive = "Zenodo"; - } - // or as institution for reports - else if (item.itemType == "report" && item.institution == "Zenodo") { - item.institution = ""; - } + //Don't use Zenodo as university for theses -- but use as archive + if (item.itemType == "thesis" && item.publisher == "Zenodo") { + item.publisher = ""; + item.archive = "Zenodo"; + } + // or as institution for reports + else if (item.itemType == "report" && item.institution == "Zenodo") { + item.institution = ""; + } - if (item.date) item.date = ZU.strToISO(item.date); - if (url.includes('#')) { - url = url.substring(0, url.indexOf('#')); - } - item.url = url; - if (abstract) item.abstractNote = abstract; + if (item.date) item.date = ZU.strToISO(item.date); + if (url.includes('#')) { + url = url.substring(0, url.indexOf('#')); + } + item.url = url; + if (abstract) item.abstractNote = abstract; - item.itemID = ""; - item.complete(); - }); - trans.translate(); + item.itemID = ""; + item.complete(); }); + trans.translate(); } /** BEGIN TEST CASES **/ var testCases = [ { "type": "web", - "url": "https://zenodo.org/record/54766?ln=en#.ZEAfIMQpAUE", - "detectedItemType": "thesis", + "url": "https://zenodo.org/records/54766#.ZEAfIMQpAUE", "items": [ { "itemType": "thesis", @@ -239,7 +234,7 @@ var testCases = [ "archive": "Zenodo", "extra": "DOI: 10.5281/zenodo.54766", "libraryCatalog": "Zenodo", - "url": "https://zenodo.org/record/54766?ln=en", + "url": "https://zenodo.org/records/54766", "attachments": [ { "title": "Zenodo Full Text PDF", @@ -258,7 +253,7 @@ var testCases = [ }, { "type": "web", - "url": "https://zenodo.org/record/54747", + "url": "https://zenodo.org/records/54747", "items": [ { "itemType": "presentation", @@ -273,7 +268,7 @@ var testCases = [ "date": "2015-09-17", "abstractNote": "Guides you through important steps in developing relevant visualizations by showcasing the work of PASTEUR4OA to develop visualizations from ROARMAP.", "extra": "DOI: 10.5281/zenodo.54747", - "url": "https://zenodo.org/record/54747", + "url": "https://zenodo.org/records/54747", "attachments": [ { "title": "Zenodo Snapshot", @@ -281,20 +276,34 @@ var testCases = [ } ], "tags": [ - "Data visualisation", - "Open Access", - "Open Access policy", - "PASTEUR4OA", - "ROARMAP" + { + "tag": "Data visualisation" + }, + { + "tag": "Open Access" + }, + { + "tag": "Open Access policy" + }, + { + "tag": "PASTEUR4OA" + }, + { + "tag": "ROARMAP" + } + ], + "notes": [ + { + "note": "Funding by European Commission ROR 00k4n6c32." + } ], - "notes": [], "seeAlso": [] } ] }, { "type": "web", - "url": "https://zenodo.org/record/14837?ln=en", + "url": "https://zenodo.org/records/14837", "items": [ { "itemType": "artwork", @@ -316,7 +325,7 @@ var testCases = [ "extra": "DOI: 10.5281/zenodo.14837", "libraryCatalog": "Zenodo", "shortTitle": "Figures 8-11 in A new Savignia from Cretan caves (Araneae", - "url": "https://zenodo.org/record/14837?ln=en", + "url": "https://zenodo.org/records/14837", "attachments": [ { "title": "Zenodo Snapshot", @@ -324,28 +333,42 @@ var testCases = [ } ], "tags": [ - "Arachnida", - "Araneae", - "Crete", - "Greece", - "Linyphiidae", - "Savignia", - "cave", - "new species", - "troglobiont" - ], - "notes": [ { - "note": "Figure uploaded by Plazi" + "tag": "Arachnida" + }, + { + "tag": "Araneae" + }, + { + "tag": "Crete" + }, + { + "tag": "Greece" + }, + { + "tag": "Linyphiidae" + }, + { + "tag": "Savignia" + }, + { + "tag": "cave" + }, + { + "tag": "new species" + }, + { + "tag": "troglobiont" } ], + "notes": [], "seeAlso": [] } ] }, { "type": "web", - "url": "https://zenodo.org/record/11879?ln=en", + "url": "https://zenodo.org/records/11879", "items": [ { "itemType": "book", @@ -364,7 +387,7 @@ var testCases = [ "libraryCatalog": "Zenodo", "place": "Düsseldorf", "publisher": "Düsseldorf University Press", - "url": "https://zenodo.org/record/11879?ln=en", + "url": "https://zenodo.org/records/11879", "attachments": [ { "title": "Zenodo Full Text PDF", @@ -372,10 +395,18 @@ var testCases = [ } ], "tags": [ - "computational linguistics", - "historical linguistics", - "phonetic alignment", - "sequence comparison" + { + "tag": "computational linguistics" + }, + { + "tag": "historical linguistics" + }, + { + "tag": "phonetic alignment" + }, + { + "tag": "sequence comparison" + } ], "notes": [], "seeAlso": [] @@ -384,8 +415,7 @@ var testCases = [ }, { "type": "web", - "url": "https://zenodo.org/record/45756?ln=en#.ZEAe1sQpAUE", - "detectedItemType": "dataset", + "url": "https://zenodo.org/records/45756#.ZEAe1sQpAUE", "items": [ { "itemType": "dataset", @@ -427,7 +457,7 @@ var testCases = [ "abstractNote": "This submission includes a tar archive of bzipped diffraction images recorded with the ADSC Q315r detector at the Advanced Photon Source of Argonne National Laboratory, Structural Biology Center beam line 19-ID. Relevant meta data can be found in the headers of those diffraction images. Please find below the content of an input file XDS.INP for the program XDS (Kabsch, 2010), which may be used for data reduction. The \"NAME_TEMPLATE_OF_DATA_FRAMES=\" item inside XDS.INP may need to be edited to point to the location of the downloaded and untarred images. !!! Paste lines below in to a file named XDS.INP DETECTOR=ADSC  MINIMUM_VALID_PIXEL_VALUE=1  OVERLOAD= 65000 DIRECTION_OF_DETECTOR_X-AXIS= 1.0 0.0 0.0 DIRECTION_OF_DETECTOR_Y-AXIS= 0.0 1.0 0.0 TRUSTED_REGION=0.0 1.05 MAXIMUM_NUMBER_OF_JOBS=10 ORGX=   1582.82  ORGY=   1485.54 DETECTOR_DISTANCE= 150 ROTATION_AXIS= -1.0 0.0 0.0 OSCILLATION_RANGE=1 X-RAY_WAVELENGTH= 1.2821511 INCIDENT_BEAM_DIRECTION=0.0 0.0 1.0 FRACTION_OF_POLARIZATION=0.90 POLARIZATION_PLANE_NORMAL= 0.0 1.0 0.0 SPACE_GROUP_NUMBER=20 UNIT_CELL_CONSTANTS= 100.030   121.697    56.554    90.000    90.000    90.000 DATA_RANGE=1  180 BACKGROUND_RANGE=1 6 SPOT_RANGE=1 3 SPOT_RANGE=31 33 MAX_CELL_AXIS_ERROR=0.03 MAX_CELL_ANGLE_ERROR=2.0 TEST_RESOLUTION_RANGE=8.0 3.8 MIN_RFL_Rmeas= 50 MAX_FAC_Rmeas=2.0 VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS= 6000 30000 INCLUDE_RESOLUTION_RANGE=50.0 1.7 FRIEDEL'S_LAW= FALSE STARTING_ANGLE= -100      STARTING_FRAME=1 NAME_TEMPLATE_OF_DATA_FRAMES= ../x247398/t1.0???.img !!! End of XDS.INP", "libraryCatalog": "Zenodo", "repository": "Zenodo", - "url": "https://zenodo.org/record/45756?ln=en", + "url": "https://zenodo.org/records/45756", "attachments": [ { "title": "Zenodo Snapshot", @@ -448,24 +478,20 @@ var testCases = [ "tag": "protein structure" } ], - "notes": [ - { - "note": "Argonne is operated by UChicago Argonne, LLC, for the U.S. Department of Energy, Office of Biological and Environmental Research under contract DE-AC02-06CH11357. The SGC is a registered charity (number 1097737) that receives funds from AbbVie, Bayer Pharma AG, Boehringer Ingelheim, Canada Foundation for Innovation, Eshelman Institute for Innovation, Genome Canada through Ontario Genomics Institute, Innovative Medicines Initiative (EU/EFPIA) [ULTRA-DD grant no. 115766], Janssen, Merck & Co., Novartis Pharma AG, Ontario Ministry of Economic Development and Innovation, Pfizer, São Paulo Research Foundation-FAPESP, Takeda, and the Wellcome Trust." - } - ], + "notes": [], "seeAlso": [] } ] }, { "type": "web", - "defer": true, "url": "https://zenodo.org/search?page=1&size=20&q=&type=video", + "defer": true, "items": "multiple" }, { "type": "web", - "url": "https://zenodo.org/record/569323", + "url": "https://zenodo.org/records/569323", "items": [ { "itemType": "journalArticle", @@ -484,7 +510,7 @@ var testCases = [ "libraryCatalog": "Zenodo", "pages": "52-56", "publicationTitle": "Qualitative & Multi-Method Research", - "url": "https://zenodo.org/record/569323", + "url": "https://zenodo.org/records/569323", "volume": "14", "attachments": [ { @@ -504,7 +530,7 @@ var testCases = [ }, { "type": "web", - "url": "https://zenodo.org/record/1048320", + "url": "https://zenodo.org/records/1048320", "items": [ { "itemType": "computerProgram", @@ -513,27 +539,27 @@ var testCases = [ { "firstName": "Carl", "lastName": "Boettiger", - "creatorType": "author" + "creatorType": "programmer" }, { "firstName": "Maëlle", "lastName": "Salmon", - "creatorType": "author" + "creatorType": "programmer" }, { "firstName": "Noam", "lastName": "Ross", - "creatorType": "author" + "creatorType": "programmer" }, { "firstName": "Arfon", "lastName": "Smith", - "creatorType": "author" + "creatorType": "programmer" }, { "firstName": "Anna", "lastName": "Krystalli", - "creatorType": "author" + "creatorType": "programmer" } ], "date": "2017-11-13", @@ -542,7 +568,8 @@ var testCases = [ "extra": "DOI: 10.5281/zenodo.1048320", "libraryCatalog": "Zenodo", "shortTitle": "ropensci/codemetar", - "url": "https://zenodo.org/record/1048320", + "url": "https://zenodo.org/records/1048320", + "versionNumber": "0.1.2", "attachments": [ { "title": "Zenodo Snapshot", @@ -554,6 +581,75 @@ var testCases = [ "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://zenodo.org/records/8092340", + "items": [ + { + "itemType": "preprint", + "title": "Creating Virtuous Cycles for DNA Barcoding: A Case Study in Science Innovation, Entrepreneurship, and Diplomacy", + "creators": [ + { + "lastName": "Schindel", + "firstName": "David E.", + "creatorType": "author" + }, + { + "lastName": "Page", + "firstName": "Roderic D. M. Page", + "creatorType": "author" + } + ], + "date": "2023-06-28", + "DOI": "10.5281/zenodo.8092340", + "abstractNote": "This essay on the history of the DNA barcoding enterprise attempts to set the stage for the more scholarly contributions that follow. How did the enterprise begin? What were its goals, how did it develop, and to what degree are its goals being realized? We have taken a keen interest in the barcoding movement and its relationship to taxonomy, collections and biodiversity informatics more broadly considered. This essay integrates our two different perspectives on barcoding. DES was the Executive Secretary of the Consortium for the Barcode of Life from 2004 to 2017, with the mission to support the success of DNA barcoding without being directly involved in generating barcode data. RDMP viewed barcoding as an important entry into the landscape of biodiversity data, with many potential linkages to other components of the landscape. We also saw it as a critical step toward the era of international genomic research that was sure to follow. Like the Mercury Program that paved the way for lunar landings by the Apollo Program, we saw DNA barcoding as the proving grounds for the interdisciplinary and international cooperation that would be needed for success of whole-genome research.", + "libraryCatalog": "Zenodo", + "repository": "Zenodo", + "shortTitle": "Creating Virtuous Cycles for DNA Barcoding", + "url": "https://zenodo.org/records/8092340", + "attachments": [ + { + "title": "Zenodo Full Text PDF", + "mimeType": "application/pdf" + } + ], + "tags": [ + { + "tag": "BOLD" + }, + { + "tag": "Barcode of Life Data System" + }, + { + "tag": "CBoL" + }, + { + "tag": "Consortium for the Barcode of Life" + }, + { + "tag": "DNA barcoding" + }, + { + "tag": "dark taxa" + }, + { + "tag": "preprint" + }, + { + "tag": "specimen voucher" + }, + { + "tag": "taxonomy" + }, + { + "tag": "virtuous cycles" + } + ], + "notes": [], + "seeAlso": [] + } + ] } ] /** END TEST CASES **/ diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index ebfc2177b33..2bb88eb79c6 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -1,21 +1,21 @@ { "translatorID": "874d70a0-6b95-4391-a681-c56dabaa1411", "label": "clinicaltrials.gov", - "creator": "Ryan Velazquez", - "target": "^https://(www\\.)?clinicaltrials\\.gov/ct2/(show|results\\?)", - "minVersion": "3.0", + "creator": "Ryan Velazquez and contributors", + "target": "^https://(classic\\.clinicaltrials\\.gov/ct2/(show|results)|(www\\.)?clinicaltrials\\.gov/(study|search))\\b", + "minVersion": "5.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2020-12-13 03:28:31" + "lastUpdated": "2023-09-17 06:35:36" } /* ***** BEGIN LICENSE BLOCK ***** - Copyright © 2020 Ryan Velazquez + Copyright © 2020 Ryan Velazquez and contributors This file is part of Zotero. @@ -36,180 +36,154 @@ */ +function isClassic(url) { + return url.startsWith("https://classic.clinicaltrials.gov"); +} + function detectWeb(doc, url) { - if (url.includes("/ct2/results?")) { - Zotero.monitorDOMChanges(doc.querySelector("#theDataTable")); - if (getSearchResults(doc, true)) { - return "multiple"; + let urlObject = new URL(url); + + if (!isClassic(url)) { // new UI and default since July 2023 + if (urlObject.pathname === "/search") { + // Watch for disappearance/appearance of results due to filtering + let resultsNode = doc.querySelector(".results-content-area"); + if (resultsNode) { + // after the node has been generated by ajax, watch it + Zotero.monitorDOMChanges(resultsNode); + } + + if (getSearchResults(doc, true/* checkOnly */)) { + return "multiple"; + } + } + else if (urlObject.pathname.startsWith("/study/")) { + return "report"; } + return false; } - if (url.includes("/ct2/show")) { + // for "classic" UI + if (urlObject.pathname === "/ct2/results") { + // The following node being watched is present on the intial page. + Zotero.monitorDOMChanges(doc.querySelector("#docListBlock")); + if (getSearchResults(doc, true/* checkOnly */)) { + return "multiple"; + } + } + else if (urlObject.pathname.startsWith("/ct2/show/")) { return "report"; } return false; } +// The keys in the returned item will be the NCTId, which is enough for +// identifying a report function getSearchResults(doc, checkOnly) { + let resultSelector + = isClassic(doc.location.href) + ? '#theDataTable a[href^="/ct2/show/"]' + : "ctg-search-hit-card header > a[href^='/study/']"; var items = {}; var found = false; - var rows = doc.querySelectorAll('table#theDataTable a[href*="/ct2/show"]'); + var rows = doc.querySelectorAll(resultSelector); for (let row of rows) { - let href = row.href; + let id = getClinicalTrialID(row.href); let title = ZU.trimInternal(row.textContent); - if (!href || !title) continue; + if (!id || !title) continue; if (checkOnly) return true; found = true; - items[href] = title; + items[id] = title; } return found ? items : false; } -function doWeb(doc, url) { +async function doWeb(doc, url) { if (detectWeb(doc, url) == "multiple") { - Zotero.selectItems(getSearchResults(doc, false), function (items) { - if (items) ZU.processDocuments(Object.keys(items), scrape); - }); - } - else { - scrape(doc, url); - } -} - -function isJsonAPIRequest(url) { - if ( - url.includes("https://clinicaltrials.gov/api/query") - && url.includes("fmt=JSON") - ) { - return true; + let trialIDs = await Z.selectItems(getSearchResults(doc)); + if (!trialIDs) return; + for (let id of Object.keys(trialIDs)) { + await scrape(id); + } } else { - return false; + await scrape(getClinicalTrialID(url)); } } -function isXmlAPIRequest(url) { - if ( - url.includes("https://clinicaltrials.gov/api/query") - && url.includes("fmt=XML") - ) { - return true; - } - else { - return false; - } +async function scrape(clinicalTrialID) { + let jsonRequestURL = `https://classic.clinicaltrials.gov/api/query/full_studies?expr=${clinicalTrialID}&fmt=JSON`; + classicJSONToItem(await requestJSON(jsonRequestURL)); } function getClinicalTrialID(url) { - if (isXmlAPIRequest(url) || isJsonAPIRequest(url)) { - return url.split("expr=")[1].split("&")[0]; - } - else { - return url.split("/show/")[1].split("?")[0]; - } + let pathComponents = new URL(url).pathname.split("/"); + return pathComponents[pathComponents.length - 1]; // last component in pathname } -function dateTimeToDateString(dateTime) { - return dateTime.split(" ")[0].split(":").join("-"); -} +// Convert the data object from the JSON response of "classic" API into item +// TODO: +// - Add support for new API +// - This can be the basis for a search and import translator +function classicJSONToItem(data) { + let item = new Zotero.Item("report"); -function scrape(doc, url) { - const clinicalTrialID = getClinicalTrialID(url); - let jsonRequestURL; - if (!isJsonAPIRequest(url)) { - jsonRequestURL = `https://clinicaltrials.gov/api/query/full_studies?expr=${clinicalTrialID}&fmt=JSON`; - } - else { - jsonRequestURL = url; - } + let study = data.FullStudiesResponse.FullStudies[0].Study; - ZU.doGet(jsonRequestURL, function (resp) { - const data = JSON.parse(resp); - var item = new Zotero.Item("report"); - const study = data.FullStudiesResponse.FullStudies[0].Study; - item.itemType = "report"; - item.title = study.ProtocolSection.IdentificationModule.OfficialTitle; + // Start get the creator info + let creators = []; - // Start get the creator info - let creators = []; - let responsiblePartyInvestigator; - let sponsor; - let collaborators = []; - if ( - study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty( - "ResponsibleParty" - ) - ) { - const responsibleParty - = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty; - if ( - typeof responsibleParty.ResponsiblePartyInvestigatorFullName == "string" - ) { - responsiblePartyInvestigator - = responsibleParty.ResponsiblePartyInvestigatorFullName; - creators.push( - ZU.cleanAuthor(responsiblePartyInvestigator, "author", false) - ); - } - } + let authorModule = study.ProtocolSection.SponsorCollaboratorsModule; + let firstAuthor = authorModule.ResponsibleParty; + let leadSponsor = authorModule.LeadSponsor; - if ( - study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty( - "LeadSponsor" - ) - ) { - sponsor - = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor - .LeadSponsorName; - let sponsorCreatorType; - if (creators.length === 0) { - sponsorCreatorType = "author"; - } - else { - sponsorCreatorType = "contributor"; - } - creators.push({ - lastName: sponsor, - creatorType: sponsorCreatorType, - fieldMode: 1 - }); - } + let investigatorName; + if (firstAuthor + && firstAuthor.ResponsiblePartyType !== "Sponsor") { // a person + // Clean up the comma trailing titles such as "First Last, MD, PhD" + investigatorName = firstAuthor.ResponsiblePartyInvestigatorFullName; + let cleanName = investigatorName.split(", ")[0]; + creators.push(ZU.cleanAuthor(cleanName, "author")); + } + + if (leadSponsor && leadSponsor.LeadSponsorName !== investigatorName) { + // lead sponsor is not a duplicate of the PI + creators.push({ + lastName: leadSponsor.LeadSponsorName, + creatorType: (creators.length ? "contributor" : "author"), + fieldMode: 1 + }); + } - if ( - study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty( - "CollaboratorList" - ) - ) { - const collaboratorList - = study.ProtocolSection.SponsorCollaboratorsModule.CollaboratorList - .Collaborator; - collaboratorList.forEach((collaborator) => { - collaborators.push({ - lastName: collaborator.CollaboratorName, + if (authorModule.CollaboratorList) { + let collabArray = authorModule.CollaboratorList.Collaborator || []; + for (let entity of collabArray) { + if (entity && entity.CollaboratorName) { + creators.push({ + lastName: entity.CollaboratorName, creatorType: "contributor", fieldMode: 1 }); - }); - collaborators.forEach((collaborator) => { - creators.push(collaborator); - }); + } } + } + + item.creators = creators; - item.creators = creators; - // Done get the creator info + let idModule = study.ProtocolSection.IdentificationModule; + let statusModule = study.ProtocolSection.StatusModule; - item.date = study.ProtocolSection.StatusModule.LastUpdateSubmitDate; - item.accessDate = dateTimeToDateString(data.FullStudiesResponse.DataVrs); - item.institution = "clinicaltrials.gov"; - item.reportNumber = clinicalTrialID; - item.shortTitle = study.ProtocolSection.IdentificationModule.BriefTitle; - item.abstractNote = study.ProtocolSection.DescriptionModule.BriefSummary; - item.url = "https://clinicaltrials.gov/ct2/show/" + clinicalTrialID; - item.reportType = "Clinical trial registration"; - item.extra = `submitted: ${study.ProtocolSection.StatusModule.StudyFirstSubmitDate}`; - item.complete(); - }); + item.title = idModule.OfficialTitle; + item.date = statusModule.LastUpdateSubmitDate; + item.accessDate = ZU.strToISO(data.FullStudiesResponse.DataVrs); + item.institution = "clinicaltrials.gov"; // publisher + item.reportNumber = idModule.NCTId; + item.shortTitle = idModule.BriefTitle; + item.abstractNote = study.ProtocolSection.DescriptionModule.BriefSummary; + item.url = "https://clinicaltrials.gov/study/" + idModule.NCTId; + item.reportType = "Clinical trial registration"; + item.extra = `submitted: ${statusModule.StudyFirstSubmitDate}`; + item.complete(); } @@ -217,7 +191,7 @@ function scrape(doc, url) { var testCases = [ { "type": "web", - "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", + "url": "https://clinicaltrials.gov/study/NCT04292899", "items": [ { "itemType": "report", @@ -229,7 +203,7 @@ var testCases = [ "fieldMode": 1 } ], - "date": "July 23, 2020", + "date": "December 15, 2020", "abstractNote": "The primary objective of this study is to evaluate the efficacy of 2 remdesivir (RDV) regimens with respect to clinical status assessed by a 7-point ordinal scale on Day 14.", "extra": "submitted: February 28, 2020", "institution": "clinicaltrials.gov", @@ -237,7 +211,7 @@ var testCases = [ "reportNumber": "NCT04292899", "reportType": "Clinical trial registration", "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", - "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", + "url": "https://clinicaltrials.gov/study/NCT04292899", "attachments": [], "tags": [], "notes": [], @@ -247,7 +221,7 @@ var testCases = [ }, { "type": "web", - "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", + "url": "https://clinicaltrials.gov/study/NCT00287391", "items": [ { "itemType": "report", @@ -272,7 +246,7 @@ var testCases = [ "reportNumber": "NCT00287391", "reportType": "Clinical trial registration", "shortTitle": "Sleep Disorders and Gastroesophageal Reflux Disease (GERD)", - "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", + "url": "https://clinicaltrials.gov/study/NCT00287391", "attachments": [], "tags": [], "notes": [], @@ -282,7 +256,7 @@ var testCases = [ }, { "type": "web", - "url": "https://clinicaltrials.gov/ct2/show/NCT04261517?recrs=e&cond=COVID&draw=2", + "url": "https://clinicaltrials.gov/study/NCT04261517?recrs=e&cond=COVID&draw=2", "items": [ { "itemType": "report", @@ -306,13 +280,55 @@ var testCases = [ "libraryCatalog": "clinicaltrials.gov", "reportNumber": "NCT04261517", "reportType": "Clinical trial registration", - "url": "https://clinicaltrials.gov/ct2/show/NCT04261517", + "url": "https://clinicaltrials.gov/study/NCT04261517", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://classic.clinicaltrials.gov/ct2/show/NCT04292899", + "items": [ + { + "itemType": "report", + "title": "A Phase 3 Randomized Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe COVID-19", + "creators": [ + { + "lastName": "Gilead Sciences", + "creatorType": "author", + "fieldMode": 1 + } + ], + "date": "December 15, 2020", + "abstractNote": "The primary objective of this study is to evaluate the efficacy of 2 remdesivir (RDV) regimens with respect to clinical status assessed by a 7-point ordinal scale on Day 14.", + "extra": "submitted: February 28, 2020", + "institution": "clinicaltrials.gov", + "libraryCatalog": "clinicaltrials.gov", + "reportNumber": "NCT04292899", + "reportType": "Clinical trial registration", + "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", + "url": "https://clinicaltrials.gov/study/NCT04292899", "attachments": [], "tags": [], "notes": [], "seeAlso": [] } ] + }, + { + "type": "web", + "url": "https://www.clinicaltrials.gov/search?term=transgender%20care", + "defer": true, + "items": "multiple" + }, + { + "type": "web", + "url": "https://classic.clinicaltrials.gov/ct2/results?cond=Coronavirus+Disease+2019&term=&cntry=&state=&city=&dist=&Search=Search", + "defer": true, + "items": "multiple" } ] /** END TEST CASES **/ diff --git a/eLibrary.ru.js b/eLibrary.ru.js index fad8aea418e..2164bfec58c 100644 --- a/eLibrary.ru.js +++ b/eLibrary.ru.js @@ -8,8 +8,8 @@ "priority": 100, "inRepository": true, "translatorType": 4, - "browserSupport": "gcsbv", - "lastUpdated": "2020-03-09 18:50:52" + "browserSupport": "gcsibv", + "lastUpdated": "2023-04-20 20:02:47" } /* @@ -182,7 +182,6 @@ function scrape(doc, url) { Номер: "issue", ISSN: "ISSN", "Число страниц": "pages", // e.g. "83" - Страницы: "pages", // e.g. "10-16" Язык: "language", "Место издания": "place" }; @@ -194,6 +193,9 @@ function scrape(doc, url) { item[mapping[key]] = t; } } + + var pages = ZU.xpathText(datablock, '//tr/td/div/text()[contains(., "Страницы")]/following-sibling::*[1]'); + if (pages) item.pages = pages; /* // Times-cited in Russian-Science-Citation-Index. @@ -202,6 +204,8 @@ function scrape(doc, url) { var rsci = ZU.xpathText(doc, '//tr/td/text()[contains(., "Цитирований в РИНЦ")]/following-sibling::*[2]'); Zotero.debug("Russian Science Citation Index: " + rsci); if (rsci) item.extra = "Цитируемость в РИНЦ: " + rsci; + + */ var journalBlock = ZU.xpath(datablock, './table/tbody[tr[1]/td/font[contains(text(), "ЖУРНАЛ:")]]/tr[2]/td[2]'); @@ -290,7 +294,7 @@ var testCases = [ }, { "type": "web", - "url": "https://elibrary.ru/item.asp?id=17339044", + "url": "https://www.elibrary.ru/item.asp?id=17339044", "items": [ { "itemType": "journalArticle", @@ -330,21 +334,9 @@ var testCases = [ "libraryCatalog": "eLibrary.ru", "pages": "1-10", "publicationTitle": "Плодоводство И Виноградарство Юга России", - "url": "https://elibrary.ru/item.asp?id=17339044", + "url": "https://www.elibrary.ru/item.asp?id=17339044", "attachments": [], "tags": [ - { - "tag": "Apple-Tree" - }, - { - "tag": "Immunity" - }, - { - "tag": "Scab" - }, - { - "tag": "Variety" - }, { "tag": "Иммунитет" }, @@ -365,7 +357,7 @@ var testCases = [ }, { "type": "web", - "url": "https://elibrary.ru/item.asp?id=21640363", + "url": "https://www.elibrary.ru/item.asp?id=21640363", "items": [ { "itemType": "journalArticle", @@ -398,7 +390,7 @@ var testCases = [ "language": "ru", "libraryCatalog": "eLibrary.ru", "pages": "83", - "url": "https://elibrary.ru/item.asp?id=21640363", + "url": "https://www.elibrary.ru/item.asp?id=21640363", "attachments": [], "tags": [], "notes": [], @@ -647,7 +639,7 @@ var testCases = [ }, { "type": "web", - "url": "https://elibrary.ru/item.asp?id=18310800", + "url": "https://www.elibrary.ru/item.asp?id=18310800", "items": [ { "itemType": "journalArticle", @@ -670,8 +662,7 @@ var testCases = [ }, { "lastName": "Де Щулф А.", - "creatorType": "author", - "fieldMode": true + "creatorType": "author" }, { "firstName": "Е.", @@ -685,8 +676,7 @@ var testCases = [ }, { "lastName": "Ван Хооф Л.", - "creatorType": "author", - "fieldMode": true + "creatorType": "author" }, { "firstName": "С.", @@ -695,8 +685,7 @@ var testCases = [ }, { "lastName": "Де Лангхе К.", - "creatorType": "author", - "fieldMode": true + "creatorType": "author" }, { "firstName": "А.", @@ -705,8 +694,7 @@ var testCases = [ }, { "lastName": "Ван Де Керчове Р.", - "creatorType": "author", - "fieldMode": true + "creatorType": "author" }, { "firstName": "Р.", @@ -715,8 +703,7 @@ var testCases = [ }, { "lastName": "Те Киефте Д.", - "creatorType": "author", - "fieldMode": true + "creatorType": "author" } ], "date": "2009", @@ -726,32 +713,107 @@ var testCases = [ "libraryCatalog": "eLibrary.ru", "pages": "10-20", "publicationTitle": "Мир Евразии", - "url": "https://elibrary.ru/item.asp?id=18310800", + "url": "https://www.elibrary.ru/item.asp?id=18310800", "attachments": [], "tags": [ { - "tag": "Belgian-Russian Expedition" + "tag": "Бельгийско-Русская Экспедиция" }, { - "tag": "Karakol" + "tag": "Каракол" }, { - "tag": "Scythian Culture" + "tag": "Парк Уч-Энмек" }, { - "tag": "Uch Enmek Park" + "tag": "Скифская Культура" + } + ], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://www.elibrary.ru/item.asp?id=22208210", + "items": [ + { + "itemType": "journalArticle", + "title": "Biological and cognitive correlates of murder and attempted murder in the italian regions", + "creators": [ + { + "firstName": "D. I.", + "lastName": "Templer", + "creatorType": "author" + } + ], + "date": "2013", + "ISSN": "0025-2344", + "issue": "1", + "language": "en", + "libraryCatalog": "eLibrary.ru", + "pages": "26-48", + "publicationTitle": "Mankind Quarterly", + "url": "https://www.elibrary.ru/item.asp?id=22208210", + "volume": "54", + "attachments": [], + "tags": [], + "notes": [], + "seeAlso": [] + } + ] + }, + { + "type": "web", + "url": "https://elibrary.ru/item.asp?id=35209757", + "items": [ + { + "itemType": "journalArticle", + "title": "Факторы Патогенности Недифтерийных Коринебактерий, Выделенных От Больных С Патологией Респираторного Тракта", + "creators": [ + { + "firstName": "А. А.", + "lastName": "Алиева", + "creatorType": "author" }, { - "tag": "Бельгийско-Русская Экспедиция" + "firstName": "Галина Георгиевна", + "lastName": "Харсеева", + "creatorType": "author" }, { - "tag": "Каракол" + "firstName": "Э. О.", + "lastName": "Мангутов", + "creatorType": "author" }, { - "tag": "Парк Уч-Энмек" + "firstName": "С. Н.", + "lastName": "Головин", + "creatorType": "author" + } + ], + "date": "2018", + "DOI": "10.18821/0869-2084-2018-63-6-375-378", + "ISSN": "0869-2084, 2412-1320", + "abstractNote": "Недифтерийные коринебактерии штаммов C. pseudodiphtheriticum, несмотря на отсутствие способности продуцировать токсин, могут быть связаны с развитием воспалительных заболеваний респираторного и урогенитального тракта, кожи, гнойно-септических процессов различной локализации и др. Это свидетельствует о наличии у них факторов патогенности, помимо токсина, которые могут обусловливать адгезивную и инвазивную активность. Цель исследования - характеристика факторов патогенности (адгезивности, инвазивности) недифтерийных коринебактерий, выделенных от больных с патологией респираторного тракта. Исследованы штаммы недифтерийных коринебактерий (n = 38), выделенные из верхних дыхательных путей от больных с хроническим тонзиллитом (C. pseudodiphtheriticum, n = 9 ), ангинами (C. pseudodiphtheriticum, n = 14), практически здоровых обследованных (C. Pseudodiphtheriticum, n = 15). Способность к адгезии и инвазии коринебактерий исследовали на культуре клеток карциномы фарингеального эпителия Hep-2...\n\nНедифтерийные коринебактерии штаммов C. pseudodiphtheriticum, несмотря на отсутствие способности продуцировать токсин, могут быть связаны с развитием воспалительных заболеваний респираторного и урогенитального тракта, кожи, гнойно-септических процессов различной локализации и др. Это свидетельствует о наличии у них факторов патогенности, помимо токсина, которые могут обусловливать адгезивную и инвазивную активность. Цель исследования - характеристика факторов патогенности (адгезивности, инвазивности) недифтерийных коринебактерий, выделенных от больных с патологией респираторного тракта. Исследованы штаммы недифтерийных коринебактерий (n = 38), выделенные из верхних дыхательных путей от больных с хроническим тонзиллитом (C. pseudodiphtheriticum, n = 9 ), ангинами (C. pseudodiphtheriticum, n = 14), практически здоровых обследованных (C. Pseudodiphtheriticum, n = 15). Способность к адгезии и инвазии коринебактерий исследовали на культуре клеток карциномы фарингеального эпителия Hep-2. Количество коринебактерий, адгезированных и инвазированных на клетках Нер-2, определяли путём высева смыва на 20%-ный сывороточный агар с последующим подсчётом среднего количества колониеобразующих единиц (КОЕ) в 1 мл. Электронно-микроскопическое исследование адгезии и инвазии коринебактерий на культуре клеток Нер-2 проводили методом трансмиссионной электронной микроскопии. У выделенных от практически здоровых лиц штаммов C. pseudodiphtheriticum адгезивность была ниже (р ≤ 0,05), чем у всех исследованных штаммов недифтерийных коринебактерий, выделенных от больных с патологией респираторного тракта. Наиболее выраженные адгезивные свойства (238,3 ± 6,5 КОЕ/мл) обнаружены у штаммов C. pseudodiphtheriticum, выделенных от больных ангинами, по сравнению с таковыми, выделенными от больных хроническим тонзиллитом. Адгезивность и инвазивность у всех исследованных штаммов имели положительную коррелятивную связь. При электронно-микроскопическом исследовании видны коринебактерии, как адгезированные на поверхности клеток Нер-2 и накопившие контрастное вещество, так и инвазированные, электронно-прозрачные. Недифтерийные коринебактерии штаммов С. pseudodiphtheriticum, выделенных от больных с патологией респираторного тракта (ангина, хронический тонзиллит), обладали более высокой способностью к адгезии и инвазии по сравнению со штаммами С. pseudodiphtheriticum, изолированными от практически здоровых лиц. Выраженная способность к адгезии и инвазии, рассматриваемым как факторы патогенности С.pseudodiphtheriticum, позволяет им реализовывать свой патогенный потенциал, защищая от действия иммунной системы хозяина и антибактериальных препаратов.\n\nfunction show_abstract() {\n $('#abstract1').hide();\n $('#abstract2').show();\n $('#abstract_expand').hide();\n}\n\n▼Показать полностью", + "issue": "6", + "language": "ru", + "libraryCatalog": "eLibrary.ru", + "pages": "375-378", + "publicationTitle": "Клиническая Лабораторная Диагностика", + "url": "https://elibrary.ru/item.asp?id=35209757", + "volume": "63", + "attachments": [], + "tags": [ + { + "tag": "Адгезия" }, { - "tag": "Скифская Культура" + "tag": "Инвазия" + }, + { + "tag": "Факторы Патогенности" } ], "notes": [], diff --git a/ePrint IACR.js b/ePrint IACR.js index 120aab53dc7..3cecce507e2 100644 --- a/ePrint IACR.js +++ b/ePrint IACR.js @@ -2,115 +2,223 @@ "translatorID": "04a23cbe-5f8b-d6cd-8eb1-2e23bcc8ae8f", "label": "ePrint IACR", "creator": "Jonas Schrieb", - "target": "^https?://eprint\\.iacr\\.org/", - "minVersion": "1.0.0b3.r1", + "target": "^https://eprint\\.iacr\\.org/", + "minVersion": "6.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2022-10-28 16:15:17" + "lastUpdated": "2023-07-22 15:29:56" } -let preprintType = ZU.fieldIsValidForType('title', 'preprint') - ? 'preprint' - : 'report'; +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2010-2023 Jonas Schrieb and contributors + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see <http://www.gnu.org/licenses/>. + + ***** END LICENSE BLOCK ***** +*/ + +// There are several types of searches available on ePrint, producing pages with different structure, need to distinguish them +// This covers the standard text-based search +const SEARCH_TYPE_TEXT = "text"; +// This covers all time-based searches: "All papers" (excluding "Compact view"), "Updates from the last X days", "Listing by year" +const SEARCH_TYPE_TIME = "time"; +let searchType = ""; function detectWeb(doc, url) { - var singleRe = /^https?:\/\/eprint\.iacr\.org\/(\d{4}\/\d{3}|cgi-bin\/print\.pl)/; - var multipleRe = /^https?:\/\/eprint\.iacr\.org\/search\?/; + var eprintBaseURLRe = /^https:\/\/eprint\.iacr\.org\//; + // Single paper URL format is https://<ePrint FQDN>/<year>/<paper number within the year> + // The year is always 4 digits, paper number can technically be 1 digit or more, + // the default is 3 or 4 digits (the former is left-padded with zeroes if smaller than 100) + var singleRe = new RegExp(eprintBaseURLRe.source + /\d{4}\/\d+/.source); + + var multipleTextSearchRe = new RegExp(eprintBaseURLRe.source + /search\?/.source); + var multipleTimeSearchYearRe = new RegExp(eprintBaseURLRe.source + /\d{4}\/?$/.source); + var multipleTimeSearchDaysRe = new RegExp(eprintBaseURLRe.source + /days/.source); + // The "Compact view" is explicitly unsupported - it does not use pagination and trying to handle all 20K+ papers at once reliably kills Zotero + var multipleTimeSearchAllRe = new RegExp(eprintBaseURLRe.source + /complete\/?$/.source); + if (singleRe.test(url)) { - return preprintType; - } else if (multipleRe.test(url)) { - return "multiple"; + return 'preprint'; } + else if (multipleTextSearchRe.test(url)) { + searchType = SEARCH_TYPE_TEXT; + } + else if (multipleTimeSearchYearRe.test(url) + || multipleTimeSearchDaysRe.test(url) + || multipleTimeSearchAllRe.test(url)) { + searchType = SEARCH_TYPE_TIME; + } + if (searchType && getSearchResults(doc, true)) return "multiple"; + + return false; } -function scrape(doc, url) { - var reportNoSelector = "h4"; - var titleSelector = "h3"; - var authorsSelector = 'meta[name="citation_author"]' - var abstractXPath = "//h5[starts-with(text(),\"Abstract\")]/following-sibling::p/text()"; +async function scrape(doc, url = doc.location.href) { + var titleSelector = 'meta[name="citation_title"]'; + var authorsSelector = 'meta[name="citation_author"]'; + var archiveSelector = 'meta[name="citation_journal_title"]'; + var abstractXPath = "//h5[starts-with(text(),'Abstract')]/following-sibling::p[1]"; + var noteXPath = "//h5[starts-with(text(),'Abstract')]/following-sibling::p[2]/strong[starts-with(text(), 'Note:')]/.."; var keywordsSelector = ".keywords > .keyword"; - var reportNo = text(doc, reportNoSelector); - reportNo = reportNo.match(/(\d{4})\/(\d{3,4})$/); - if (reportNo){ - var year = reportNo[1]; - var no = reportNo[2]; + var publicationInfoSelector = "//div[@id='metadata']/dl/dt[starts-with(text(), 'Publication info')]/following-sibling::dd[1]"; + var paperIDSelector = "#eprintContent h4"; + var paperID = text(doc, paperIDSelector); + // The year is always 4 digits, the paper number is canonicalized (3 or 4 digits) before calling scrape() + paperID = paperID.match(/(\d{4})\/(\d{3,4})$/); + if (paperID) { + var paperYear = paperID[1]; + var paperNum = paperID[2]; + paperID = paperYear + "/" + paperNum; } - var title = text(doc, titleSelector); - title = ZU.trimInternal(title); + var title = ZU.trimInternal(attr(doc, titleSelector, 'content')); + + var archiveName = ZU.trimInternal(attr(doc, archiveSelector, 'content')); var authors = doc.querySelectorAll(authorsSelector); authors = [...authors].map(author => author.content); - - var abstr = ""; - var abstractLines = doc.evaluate(abstractXPath, doc, null, XPathResult.ANY_TYPE, null); - var nextLine; - while (nextLine = abstractLines.iterateNext()) { - // An inner line starting with \n starts a new paragraph in the abstract. - if (nextLine.textContent[0] == "\n") { - abstr += "\n\n"; - } - abstr += ZU.trimInternal(nextLine.textContent); - } - + + var abstr = ZU.xpathText(doc, abstractXPath); + // Remove surplus whitespace, but preserve paragraphs, denoted in the page markup by double newlines with some spaces in between + if (abstr) abstr = abstr.replace(/\n\s+\n/g, "\n\n").replace(/[ \t]+/g, " ").trim(); + + let note = ZU.xpathText(doc, noteXPath); + if (note) note = ZU.trimInternal(note.replace(/^Note: /, "")); + + let publicationInfo = ZU.xpathText(doc, publicationInfoSelector); + publicationInfo = ZU.trimInternal(publicationInfo); + var keywords = doc.querySelectorAll(keywordsSelector); keywords = [...keywords].map(kw => kw.textContent.trim()); - var newItem = new Zotero.Item(preprintType); - - newItem.date = year; - newItem.reportNumber = no; - //we want to use this later & make sure we don't make http--> https requests or vice versa. - newItem.url = url.match(/^https?:\/\/[^\/]+/)[0] + "/" + year + "/" + no; + var newItem = new Zotero.Item('preprint'); + + newItem.date = paperYear; + + let urlComponents = url.match(/^https:\/\/([^/]+)/); + let eprintFQDN = urlComponents[1]; + // TODO: Do we want to differentiate Archive and Library Catalog like this (the latter has FQDN appended)? Do we want to populate both or just one of them? The translator population has all possible approaches. + newItem.archive = archiveName; + newItem.libraryCatalog = `${archiveName} (${eprintFQDN})`; + newItem.archiveID = paperID; + + // Canonicalize the URL to avoid errors if e.g., the user removed or prepended extra zeroes to the paper ID in the original URL + newItem.url = urlComponents[0] + "/" + paperID; newItem.title = title; newItem.abstractNote = abstr; - for (var i in authors) { - newItem.creators.push(Zotero.Utilities.cleanAuthor(authors[i], "author")); + + for (let i in authors) { + newItem.creators.push(ZU.cleanAuthor(authors[i], "author")); } -for (var i = 0; i < keywords.length; i++) { - //sometimes the keywords split returns an empty tag - those crash the translator if they're pushed. - if (keywords[i] != null){ - newItem.tags.push(keywords[i]);} + + for (let i in keywords) { + newItem.tags.push(keywords[i]); } + newItem.attachments = [ - {url:newItem.url+".pdf", title:"Full Text PDF", mimeType:"application/pdf"} + { url: newItem.url + ".pdf", title: "Full Text PDF", mimeType: "application/pdf" } ]; + + if (note) newItem.notes.push({ note: note }); + + let extra = `Publication info: ${publicationInfo}`; + newItem.extra = newItem.extra + ? newItem.extra + `\n${extra}` + : extra; + newItem.complete(); +} +function getSearchResults(doc, checkonly) { + if (searchType === SEARCH_TYPE_TEXT) { + return getTextSearchResults(doc, checkonly); + } + else if (searchType === SEARCH_TYPE_TIME) { + return getTimeSearchResults(doc, checkonly); + } + return false; } -function doWeb(doc, url) { - var nextTitle; +// Standard (text) search results parser +function getTextSearchResults(doc, checkOnly) { + let rowSelector = ".results > div"; + let titleSelector = "div > strong:first-child"; + let linkSelector = "a.paperlink"; - if (detectWeb(doc, url) == "multiple") { - var rowSelector = ".paperList > div, .results > div"; - var titleSelector = ".papertitle, strong"; - var linkSelector = ".paperlink, a:first-child"; - - let items = {}; - for (let row of doc.querySelectorAll(rowSelector)) { - let title = text(row, titleSelector); - let href = attr(row, linkSelector, 'href'); - if (!title || !href) continue; - items[href] = title; - } + let items = {}; + let found = false; + // Each "row" div will contain two nested divs with necessary information + for (let row of doc.querySelectorAll(rowSelector)) { + let title = text(row, titleSelector); + let href = attr(row, linkSelector, 'href'); + if (!title || !href) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +// Time-based search results parser ("All papers", "Listing by year", "Updates from the last X days") +function getTimeSearchResults(doc, checkOnly) { + let rowSelector = ".paperList > div"; + let titleSelector = ".papertitle"; + let linkSelector = "a:first-child"; - var titles = doc.querySelectorAll(titleSelector); - var links = doc.querySelectorAll(linkSelector); - Zotero.selectItems(items, function (items) { - if (items) ZU.processDocuments(Object.keys(items), scrape); - }); - } else { - if (url.search(/\.pdf$/)!= -1) { - //go to the landing page to scrape - url = url.replace(/\.pdf$/, ""); - ZU.processDocuments([url], scrape) + let items = {}; + let found = false; + // This search type returns a page with a flat list of divs, odd ones contain links, even ones contain titles + // We treat the list as a set of [virtual] rows, each comprised of a pair of divs + let rows = doc.querySelectorAll(rowSelector); + for (let i = 0; i < rows.length - 1; i += 2) { + let href = attr(rows[i], linkSelector, "href"); + let title = text(rows[i + 1], titleSelector); + if (!title || !href) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; +} + +async function doWeb(doc, url) { + if (detectWeb(doc, url) == "multiple") { + let items = await Z.selectItems(getSearchResults(doc, false)); + if (!items) return; + for (let url of Object.keys(items)) { + await scrape(await requestDocument(url)); } - else scrape(doc, url) } -}/** BEGIN TEST CASES **/ + // Firefox restrictions will prevent this convenience clause from running (Chrome works as expected) + // Standard PDF saving (and metadata retrieval) logic will be used instead + else if (/\.pdf$/.test(url)) { + // Go to the landing page to scrape + url = url.replace(/\.pdf$/, ""); + await scrape(await requestDocument(url)); + } + else { + await scrape(doc, url); + } +} + +/** BEGIN TEST CASES **/ var testCases = [ { "type": "web", @@ -132,8 +240,11 @@ var testCases = [ } ], "date": "2005", - "abstractNote": "This paper describes an adaptive-chosen-ciphertext attack on the Cipher Feedback (CFB) mode of encryption as used in OpenPGP. In most circumstances it will allow an attacker to determine 16 bits of any block of plaintext with aboutoracle queries for the initial setup work andoracle queries for each block. Standard CFB mode encryption does not appear to be affected by this attack. It applies to a particular variation of CFB used by OpenPGP. In particular it exploits an ad-hoc integrity check feature in OpenPGP which was meant as a \"quick check\" to determine the correctness of the decrypting symmetric key.", - "libraryCatalog": "ePrint IACR", + "abstractNote": "This paper describes an adaptive-chosen-ciphertext attack on the Cipher Feedback (CFB) mode of encryption as used in OpenPGP. In most circumstances it will allow an attacker to determine 16 bits of any block of plaintext with about 215 oracle queries for the initial\nsetup work and 215 oracle queries for each block. Standard CFB mode encryption does not appear to be affected by this attack. It applies to a particular variation of CFB used by OpenPGP. In particular it exploits an ad-hoc integrity check feature in OpenPGP which was meant as a \"quick check\" to determine the correctness of the decrypting symmetric key.", + "archive": "Cryptology ePrint Archive", + "archiveID": "2005/033", + "extra": "Publication info: Published elsewhere. Unknown where it was published", + "libraryCatalog": "Cryptology ePrint Archive (eprint.iacr.org)", "url": "https://eprint.iacr.org/2005/033", "attachments": [ { @@ -179,8 +290,11 @@ var testCases = [ } ], "date": "2011", - "abstractNote": "We show that homomorphic evaluation of (wide enough) arithmetic circuits can be accomplished with only polylogarithmic overhead. Namely, we present a construction of fully homomorphic encryption (FHE) schemes that for security parametercan evaluate any width-circuit withgates in time. To get low overhead, we use the recent batch homomorphic evaluation techniques of Smart-Vercauteren and Brakerski-Gentry-Vaikuntanathan, who showed that homomorphic operations can be applied to \"packed\" ciphertexts that encrypt vectors of plaintext elements. In this work, we introduce permuting/routing techniques to move plaintext elements across these vectors efficiently. Hence, we are able to implement general arithmetic circuit in a batched fashion without ever needing to \"unpack\" the plaintext vectors. We also introduce some other optimizations that can speed up homomorphic evaluation in certain cases. For example, we show how to use the Frobenius map to raise plaintext elements to powers of~at the \"cost\" of a linear operation.", - "libraryCatalog": "ePrint IACR", + "abstractNote": "We show that homomorphic evaluation of (wide enough) arithmetic circuits can be accomplished with only polylogarithmic overhead. Namely, we present a construction of fully homomorphic encryption (FHE) schemes that for security parameter \\secparam can evaluate any width-Ω(\\secparam) circuit with t gates in time t⋅polylog(\\secparam).\n\nTo get low overhead, we use the recent batch homomorphic evaluation techniques of Smart-Vercauteren and Brakerski-Gentry-Vaikuntanathan, who showed that homomorphic operations can be applied to \"packed\" ciphertexts that encrypt vectors of plaintext elements. In this work, we introduce permuting/routing techniques to move plaintext elements across\nthese vectors efficiently. Hence, we are able to implement general arithmetic circuit in a batched fashion without ever needing to \"unpack\" the plaintext vectors.\n\nWe also introduce some other optimizations that can speed up homomorphic evaluation in certain cases. For example, we show how to use the Frobenius map to raise plaintext elements to powers of~p at the \"cost\" of a linear operation.", + "archive": "Cryptology ePrint Archive", + "archiveID": "2011/566", + "extra": "Publication info: Published elsewhere. extended abstract in Eurocrypt 2012", + "libraryCatalog": "Cryptology ePrint Archive (eprint.iacr.org)", "url": "https://eprint.iacr.org/2011/566", "attachments": [ { @@ -213,62 +327,13 @@ var testCases = [ } ] }, - { - "type": "web", - "url": "https://eprint.iacr.org/2016/1013.pdf", - "items": [ - { - "itemType": "preprint", - "title": "A Formal Security Analysis of the Signal Messaging Protocol", - "creators": [], - "date": "2016", - "abstractNote": "The Signal protocol is a cryptographic messaging protocol that provides end-to-end encryption for instant messaging in WhatsApp, Wire, and Facebook Messenger among many others, serving well over 1 billion active users. Signal includes several uncommon security properties (such as \"future secrecy\" or \"post-compromise security\"), enabled by a novel technique called *ratcheting* in which session keys are updated with every message sent. We conduct a formal security analysis of Signal's initial extended triple Diffie-Hellman (X3DH) key agreement and Double Ratchet protocols as a multi-stage authenticated key exchange protocol. We extract from the implementation a formal description of the abstract protocol, and define a security model which can capture the ratcheting key update structure as a multi-stage model where there can be a tree of stages, rather than just a sequence. We then prove the security of Signal's key exchange core in our model, demonstrating several standard security properties. We have found no major flaws in the design, and hope that our presentation and results can serve as a foundation for other analyses of this widely adopted protocol.Fix omission in description of initial X3DH handshake, reorganize figures for improved presentation. Full list of changes in Appendix D.", - "libraryCatalog": "ePrint IACR", - "url": "https://eprint.iacr.org/2016/1013", - "attachments": [ - { - "title": "Full Text PDF", - "mimeType": "application/pdf" - } - ], - "tags": [ - { - "tag": "Signal" - }, - { - "tag": "authenticated key exchange" - }, - { - "tag": "future secrecy" - }, - { - "tag": "messaging" - }, - { - "tag": "multi-stage key exchange" - }, - { - "tag": "post-compromise security" - }, - { - "tag": "protocols" - }, - { - "tag": "provable security" - } - ], - "notes": [], - "seeAlso": [] - } - ] - }, { "type": "web", "url": "https://eprint.iacr.org/2022/1039", "items": [ { "itemType": "preprint", - "title": "The Limits of Provable Security Against Model Extraction", + "title": "Theoretical Limits of Provable Security Against Model Extraction by Efficient Observational Defenses", "creators": [ { "firstName": "Ari", @@ -277,8 +342,11 @@ var testCases = [ } ], "date": "2022", - "abstractNote": "Can we hope to provide provable security against model extraction attacks? As a step towards a theoretical study of this question, we unify and abstract a wide range of \"observational\" model extraction defense mechanisms -- roughly, those that attempt to detect model extraction using a statistical analysis conducted on the distribution over the adversary's queries. To accompany the abstract observational model extraction defense, which we call OMED for short, we define the notion of complete defenses -- the notion that benign clients can freely interact with the model -- and sound defenses -- the notion that adversarial clients are caught and prevented from reverse engineering the model. We then propose a system for obtaining provable security against model extraction by complete and sound OMEDs, using (average-case) hardness assumptions for PAC-learning. Our main result nullifies our proposal for provable security, by establishing a computational incompleteness theorem for the OMED: any efficient OMED for a machine learning model computable by a polynomial size decision tree that satisfies a basic form of completeness cannot satisfy soundness, unless the subexponential Learning Parity with Noise (LPN) assumption does not hold. To prove the incompleteness theorem, we introduce a class of model extraction attacks called natural Covert Learning attacks based on a connection to the Covert Learning model of Canetti and Karchmer (TCC '21), and show that such attacks circumvent any defense within our abstract mechanism in a black-box, nonadaptive way. Finally, we further expose the tension between Covert Learning and OMEDs by proving that Covert Learning algorithms require the nonexistence of provable security via efficient OMEDs. Therefore, we observe a \"win-win\" result by obtaining a characterization of the existence of provable security via efficient OMEDs by the nonexistence of natural Covert Learning algorithms.", - "libraryCatalog": "ePrint IACR", + "abstractNote": "Can we hope to provide provable security against model extraction attacks? As a step towards a theoretical study of this question, we unify and abstract a wide range of \"observational\" model extraction defenses (OMEDs) --- roughly, those that attempt to detect model extraction by analyzing the distribution over the adversary's queries. To accompany the abstract OMED, we define the notion of complete OMEDs --- when benign clients can freely interact with the model --- and sound OMEDs --- when adversarial clients are caught and prevented from reverse engineering the model. Our formalism facilitates a simple argument for obtaining provable security against model extraction by complete and sound OMEDs, using (average-case) hardness assumptions for PAC-learning, in a way that abstracts current techniques in the prior literature.\n\nThe main result of this work establishes a partial computational incompleteness theorem for the OMED: any efficient OMED for a machine learning model computable by a polynomial size decision tree that satisfies a basic form of completeness cannot satisfy soundness, unless the subexponential Learning Parity with Noise (LPN) assumption does not hold. To prove the incompleteness theorem, we introduce a class of model extraction attacks called natural Covert Learning attacks based on a connection to the Covert Learning model of Canetti and Karchmer (TCC '21), and show that such attacks circumvent any defense within our abstract mechanism in a black-box, nonadaptive way. As a further technical contribution, we extend the Covert Learning algorithm of Canetti and Karchmer to work over any \"concise\" product distribution (albeit for juntas of a logarithmic number of variables rather than polynomial size decision trees), by showing that the technique of learning with a distributional inverter of Binnendyk et al. (ALT '22) remains viable in the Covert Learning setting.", + "archive": "Cryptology ePrint Archive", + "archiveID": "2022/1039", + "extra": "Publication info: Published elsewhere. IEEE Secure and Trustworthy Machine Learning (SATML)", + "libraryCatalog": "Cryptology ePrint Archive (eprint.iacr.org)", "url": "https://eprint.iacr.org/2022/1039", "attachments": [ { @@ -297,7 +365,11 @@ var testCases = [ "tag": "Provable Security" } ], - "notes": [], + "notes": [ + { + "note": "This is an updated version. The paper has been modified to improve readability and argumentation. Some new results have been added in section 5. The previous version of the paper appeared under the title \"The Limits of Provable Security Against Model Extraction.\" The current version is the same as for publication in IEEE SATML '23, except for minor differences and formatting." + } + ], "seeAlso": [] } ] @@ -306,6 +378,21 @@ var testCases = [ "type": "web", "url": "https://eprint.iacr.org/search?q=test", "items": "multiple" + }, + { + "type": "web", + "url": "https://eprint.iacr.org/days/7", + "items": "multiple" + }, + { + "type": "web", + "url": "https://eprint.iacr.org/2021/", + "items": "multiple" + }, + { + "type": "web", + "url": "https://eprint.iacr.org/complete/", + "items": "multiple" } ] /** END TEST CASES **/ diff --git a/index.d.ts b/index.d.ts index bf099b37fba..d0bf3b1f99d 100644 --- a/index.d.ts +++ b/index.d.ts @@ -48,7 +48,6 @@ declare namespace Zotero { chunkSize: number, func: (chunk: Type[]) => RetType ): RetType[]; - function assignProps(target: any, source: any, props?: string[]): void; function rand(min: number, max: number): number; function getPageRange(pages: string): [fromPage: number, toPage: number]; function lpad(s: string, pad: string, length: number): string; @@ -91,19 +90,12 @@ declare namespace Zotero { // varDump function itemToCSLJSON(item: Zotero.Item): any | Promise<any>; function itemFromCSLJSON(item: Zotero.Item, cslItem: any): void; - function parseURL(url: string): { - fileName: string; - fileExtension: string; - fileBaseName: string; - }; - function resolveIntermediateURL(url: string): string; function stringToUTF8Array( s: string, array: number[] | Uint8Array, offset?: number ): void; function getStringByteLength(s: string): number; - function determineAttachmentIcon(attachment: Attachment): string; function generateObjectKey(): string; function isValidObjectKey(s: string): boolean; // XRegExp @@ -215,36 +207,21 @@ declare namespace Zotero { proxy?: boolean; } - type CreatorType = - | "artist" - | "contributor" - | "performer" - | "composer" - | "wordsBy" - | "sponsor" - | "cosponsor" - | "author" - | "commenter" - | "editor" - | "translator" - | "seriesEditor" - | "bookAuthor" - | "counsel" - | "programmer" - | "reviewedAuthor" - | "recipient" - | "director" - | "scriptwriter" - | "producer" - | "interviewee" - | "interviewer" - | "cartographer" - | "inventor" - | "attorneyAgent" - | "podcaster" - | "guest" - | "presenter" - | "castMember"; + interface Tag { + tag: string; + } + + type ItemType = keyof ItemTypes; + + /** + * Generic item with unknown type. + */ + type Item = ItemTypes[ItemType]; + + var Item: { + new <T extends ItemType>(itemType: T): ItemTypes[T]; + new(itemType: string): Item; + } interface Creator<T extends CreatorType> { lastName: string?; @@ -253,10 +230,7 @@ declare namespace Zotero { fieldMode: 1?; } - interface Tag { - tag: string; - } - + /* *** BEGIN GENERATED TYPES *** */ type ItemTypes = { "artwork": ArtworkItem, "audioRecording": AudioRecordingItem, @@ -267,6 +241,7 @@ declare namespace Zotero { "case": CaseItem, "computerProgram": ComputerProgramItem, "conferencePaper": ConferencePaperItem, + "dataset": DatasetItem, "dictionaryEntry": DictionaryEntryItem, "document": DocumentItem, "email": EmailItem, @@ -288,24 +263,13 @@ declare namespace Zotero { "presentation": PresentationItem, "radioBroadcast": RadioBroadcastItem, "report": ReportItem, + "standard": StandardItem, "statute": StatuteItem, "thesis": ThesisItem, "tvBroadcast": TVBroadcastItem, "videoRecording": VideoRecordingItem, "webpage": WebpageItem, - } - - type ItemType = keyof ItemTypes; - - var Item: { - new <T extends ItemType>(itemType: T): ItemTypes[T]; - new(itemType: string): Item; - } - - /** - * Generic item with unknown type. - */ - type Item = ItemTypes[ItemType]; + }; type ArtworkItem = { itemType: "artwork"; @@ -359,7 +323,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"performer" | "composer" | "contributor" | "wordsBy">[]; + creators: Creator<"performer" | "contributor" | "composer" | "wordsBy">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -389,7 +353,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"sponsor" | "contributor" | "cosponsor">[]; + creators: Creator<"sponsor" | "cosponsor" | "contributor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -448,7 +412,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "editor" | "seriesEditor" | "translator">[]; + creators: Creator<"author" | "contributor" | "editor" | "translator" | "seriesEditor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -484,7 +448,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "bookAuthor" | "contributor" | "editor" | "seriesEditor" | "translator">[]; + creators: Creator<"author" | "contributor" | "editor" | "bookAuthor" | "translator" | "seriesEditor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -512,7 +476,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "counsel">[]; + creators: Creator<"author" | "counsel" | "contributor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -579,7 +543,41 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "editor" | "seriesEditor" | "translator">[]; + creators: Creator<"author" | "contributor" | "editor" | "translator" | "seriesEditor">[]; + attachments: Attachment[]; + tags: Tag[]; + notes: Note[]; + seeAlso: string[]; + complete(): void; + + [key: string]: string; + }; + + type DatasetItem = { + itemType: "dataset"; + title?: string; + abstractNote?: string; + identifier?: string; + type?: string; + versionNumber?: string; + date?: string; + repository?: string; + repositoryLocation?: string; + format?: string; + DOI?: string; + citationKey?: string; + url?: string; + accessDate?: string; + archive?: string; + archiveLocation?: string; + shortTitle?: string; + language?: string; + libraryCatalog?: string; + callNumber?: string; + rights?: string; + extra?: string; + + creators: Creator<"author" | "contributor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -615,7 +613,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "editor" | "seriesEditor" | "translator">[]; + creators: Creator<"author" | "contributor" | "editor" | "translator" | "seriesEditor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -642,7 +640,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "editor" | "reviewedAuthor" | "translator">[]; + creators: Creator<"author" | "contributor" | "editor" | "translator" | "reviewedAuthor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -700,7 +698,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "editor" | "seriesEditor" | "translator">[]; + creators: Creator<"author" | "contributor" | "editor" | "translator" | "seriesEditor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -730,7 +728,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"director" | "contributor" | "producer" | "scriptwriter">[]; + creators: Creator<"director" | "contributor" | "scriptwriter" | "producer">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -870,7 +868,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "editor" | "reviewedAuthor" | "translator">[]; + creators: Creator<"author" | "contributor" | "editor" | "translator" | "reviewedAuthor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -928,7 +926,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "reviewedAuthor" | "translator">[]; + creators: Creator<"author" | "contributor" | "translator" | "reviewedAuthor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -1022,7 +1020,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "reviewedAuthor" | "translator">[]; + creators: Creator<"author" | "contributor" | "translator" | "reviewedAuthor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -1114,7 +1112,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "editor" | "reviewedAuthor" | "translator">[]; + creators: Creator<"author" | "contributor" | "editor" | "translator" | "reviewedAuthor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -1171,7 +1169,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"director" | "castMember" | "contributor" | "guest" | "producer" | "scriptwriter">[]; + creators: Creator<"director" | "scriptwriter" | "producer" | "castMember" | "contributor" | "guest">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -1203,7 +1201,44 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"author" | "contributor" | "seriesEditor" | "translator">[]; + creators: Creator<"author" | "contributor" | "translator" | "seriesEditor">[]; + attachments: Attachment[]; + tags: Tag[]; + notes: Note[]; + seeAlso: string[]; + complete(): void; + + [key: string]: string; + }; + + type StandardItem = { + itemType: "standard"; + title?: string; + abstractNote?: string; + organization?: string; + committee?: string; + type?: string; + number?: string; + versionNumber?: string; + status?: string; + date?: string; + publisher?: string; + place?: string; + DOI?: string; + citationKey?: string; + url?: string; + accessDate?: string; + archive?: string; + archiveLocation?: string; + shortTitle?: string; + numPages?: string; + language?: string; + libraryCatalog?: string; + callNumber?: string; + rights?: string; + extra?: string; + + creators: Creator<"author" | "contributor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -1294,7 +1329,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"director" | "castMember" | "contributor" | "guest" | "producer" | "scriptwriter">[]; + creators: Creator<"director" | "scriptwriter" | "producer" | "castMember" | "contributor" | "guest">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -1328,7 +1363,7 @@ declare namespace Zotero { rights?: string; extra?: string; - creators: Creator<"director" | "castMember" | "contributor" | "producer" | "scriptwriter">[]; + creators: Creator<"director" | "scriptwriter" | "producer" | "castMember" | "contributor">[]; attachments: Attachment[]; tags: Tag[]; notes: Note[]; @@ -1362,6 +1397,38 @@ declare namespace Zotero { [key: string]: string; }; + type CreatorType = + | "artist" + | "attorneyAgent" + | "author" + | "bookAuthor" + | "cartographer" + | "castMember" + | "commenter" + | "composer" + | "contributor" + | "cosponsor" + | "counsel" + | "director" + | "editor" + | "guest" + | "interviewee" + | "interviewer" + | "inventor" + | "performer" + | "podcaster" + | "presenter" + | "producer" + | "programmer" + | "recipient" + | "reviewedAuthor" + | "scriptwriter" + | "seriesEditor" + | "sponsor" + | "translator" + | "wordsBy"; + /* *** END GENERATED TYPES *** */ + interface Note { title?: string; note: string; diff --git a/package-lock.json b/package-lock.json index b8e7c8d2f33..22b4b9033e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "CC0-1.0", "devDependencies": { "@zotero/eslint-config": "^1.0.5", - "chromedriver": "^95.0.0", + "chromedriver": "^115.0.0", "eslint": "^8.38.0", "eslint-plugin-zotero-translator": "file:.ci/eslint-plugin-zotero-translator", "selenium-webdriver": "^4.0.0-alpha.7" @@ -223,19 +223,6 @@ "node": ">= 6.0.0" } }, - "node_modules/aggregate-error": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", - "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", - "dev": true, - "dependencies": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -282,22 +269,21 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true }, "node_modules/axios": { - "version": "0.21.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", - "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", + "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", "dev": true, "dependencies": { - "follow-redirects": "^1.14.0" + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" } }, "node_modules/balanced-match": { @@ -316,18 +302,6 @@ "concat-map": "0.0.1" } }, - "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, - "dependencies": { - "fill-range": "^7.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -363,17 +337,17 @@ } }, "node_modules/chromedriver": { - "version": "95.0.0", - "resolved": "https://registry.npmjs.org/chromedriver/-/chromedriver-95.0.0.tgz", - "integrity": "sha512-HwSg7S0ZZYsHTjULwxFHrrUqEpz1+ljDudJM3eOquvqD5QKnR5pSe/GlBTY9UU2tVFRYz8bEHYC4Y8qxciQiLQ==", + "version": "115.0.0", + "resolved": "https://registry.npmjs.org/chromedriver/-/chromedriver-115.0.0.tgz", + "integrity": "sha512-mkPL+sXMLMUenoXCiKREw+cBl3ibfhiWxkiv9ByIPpqtrrInCt9zKdOolAsbmN/ndlH51WtT5ukUKbeRdrpikg==", "dev": true, "hasInstallScript": true, "dependencies": { - "@testim/chrome-version": "^1.0.7", - "axios": "^0.21.2", - "del": "^6.0.0", + "@testim/chrome-version": "^1.1.3", + "axios": "^1.4.0", + "compare-versions": "^6.0.0", "extract-zip": "^2.0.1", - "https-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.1", "proxy-from-env": "^1.1.0", "tcp-port-used": "^1.0.1" }, @@ -381,16 +355,7 @@ "chromedriver": "bin/chromedriver" }, "engines": { - "node": ">=10" - } - }, - "node_modules/clean-stack": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", - "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", - "dev": true, - "engines": { - "node": ">=6" + "node": ">=16" } }, "node_modules/color-convert": { @@ -411,12 +376,30 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, + "node_modules/compare-versions": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-6.0.0.tgz", + "integrity": "sha512-s2MzYxfRsE9f/ow8hjn7ysa7pod1xhHdQMsgiJtKx6XSNf4x2N1KG4fjrkUmXcP/e9Y2ZX4zB6sHIso0Lm6evQ==", + "dev": true + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -466,38 +449,13 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, - "node_modules/del": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/del/-/del-6.1.1.tgz", - "integrity": "sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==", - "dev": true, - "dependencies": { - "globby": "^11.0.1", - "graceful-fs": "^4.2.4", - "is-glob": "^4.0.1", - "is-path-cwd": "^2.2.0", - "is-path-inside": "^3.0.2", - "p-map": "^4.0.0", - "rimraf": "^3.0.2", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", "dev": true, - "dependencies": { - "path-type": "^4.0.0" - }, "engines": { - "node": ">=8" + "node": ">=0.4.0" } }, "node_modules/doctrine": { @@ -707,34 +665,6 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, - "node_modules/fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fast-glob/node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -777,18 +707,6 @@ "node": "^10.12.0 || >=12.0.0" } }, - "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/find-root": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", @@ -850,6 +768,20 @@ } } }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -918,32 +850,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true - }, "node_modules/grapheme-splitter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", @@ -1012,15 +918,6 @@ "node": ">=0.8.19" } }, - "node_modules/indent-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1067,24 +964,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-path-cwd": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", - "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", - "dev": true, - "engines": { - "node": ">=6" - } - }, "node_modules/is-path-inside": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", @@ -1215,26 +1094,25 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "dev": true, "engines": { - "node": ">= 8" + "node": ">= 0.6" } }, - "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dev": true, "dependencies": { - "braces": "^3.0.2", - "picomatch": "^2.3.1" + "mime-db": "1.52.0" }, "engines": { - "node": ">=8.6" + "node": ">= 0.6" } }, "node_modules/minimatch": { @@ -1317,21 +1195,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-map": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", - "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", - "dev": true, - "dependencies": { - "aggregate-error": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", @@ -1377,33 +1240,12 @@ "node": ">=8" } }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "dev": true }, - "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -1604,15 +1446,6 @@ "node": ">=8" } }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -1703,18 +1536,6 @@ "node": ">=8.17.0" } }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -1978,16 +1799,6 @@ "debug": "4" } }, - "aggregate-error": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", - "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", - "dev": true, - "requires": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - } - }, "ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2021,19 +1832,21 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "dev": true }, "axios": { - "version": "0.21.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", - "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", + "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", "dev": true, "requires": { - "follow-redirects": "^1.14.0" + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" } }, "balanced-match": { @@ -2052,15 +1865,6 @@ "concat-map": "0.0.1" } }, - "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, - "requires": { - "fill-range": "^7.0.1" - } - }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -2084,26 +1888,20 @@ } }, "chromedriver": { - "version": "95.0.0", - "resolved": "https://registry.npmjs.org/chromedriver/-/chromedriver-95.0.0.tgz", - "integrity": "sha512-HwSg7S0ZZYsHTjULwxFHrrUqEpz1+ljDudJM3eOquvqD5QKnR5pSe/GlBTY9UU2tVFRYz8bEHYC4Y8qxciQiLQ==", + "version": "115.0.0", + "resolved": "https://registry.npmjs.org/chromedriver/-/chromedriver-115.0.0.tgz", + "integrity": "sha512-mkPL+sXMLMUenoXCiKREw+cBl3ibfhiWxkiv9ByIPpqtrrInCt9zKdOolAsbmN/ndlH51WtT5ukUKbeRdrpikg==", "dev": true, "requires": { - "@testim/chrome-version": "^1.0.7", - "axios": "^0.21.2", - "del": "^6.0.0", + "@testim/chrome-version": "^1.1.3", + "axios": "^1.4.0", + "compare-versions": "^6.0.0", "extract-zip": "^2.0.1", - "https-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.1", "proxy-from-env": "^1.1.0", "tcp-port-used": "^1.0.1" } }, - "clean-stack": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", - "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", - "dev": true - }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -2119,12 +1917,27 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, + "compare-versions": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-6.0.0.tgz", + "integrity": "sha512-s2MzYxfRsE9f/ow8hjn7ysa7pod1xhHdQMsgiJtKx6XSNf4x2N1KG4fjrkUmXcP/e9Y2ZX4zB6sHIso0Lm6evQ==", + "dev": true + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -2163,30 +1976,11 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, - "del": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/del/-/del-6.1.1.tgz", - "integrity": "sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==", - "dev": true, - "requires": { - "globby": "^11.0.1", - "graceful-fs": "^4.2.4", - "is-glob": "^4.0.1", - "is-path-cwd": "^2.2.0", - "is-path-inside": "^3.0.2", - "p-map": "^4.0.0", - "rimraf": "^3.0.2", - "slash": "^3.0.0" - } - }, - "dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "requires": { - "path-type": "^4.0.0" - } + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true }, "doctrine": { "version": "3.0.0", @@ -2345,30 +2139,6 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, - "fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "dependencies": { - "glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "requires": { - "is-glob": "^4.0.1" - } - } - } - }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -2408,15 +2178,6 @@ "flat-cache": "^3.0.4" } }, - "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, - "requires": { - "to-regex-range": "^5.0.1" - } - }, "find-root": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", @@ -2455,6 +2216,17 @@ "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", "dev": true }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -2502,26 +2274,6 @@ "type-fest": "^0.20.2" } }, - "globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "requires": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - } - }, - "graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true - }, "grapheme-splitter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/grapheme-splitter/-/grapheme-splitter-1.0.4.tgz", @@ -2572,12 +2324,6 @@ "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true }, - "indent-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true - }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -2615,18 +2361,6 @@ "is-extglob": "^2.1.1" } }, - "is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true - }, - "is-path-cwd": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", - "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", - "dev": true - }, "is-path-inside": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", @@ -2735,20 +2469,19 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "dev": true }, - "micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dev": true, "requires": { - "braces": "^3.0.2", - "picomatch": "^2.3.1" + "mime-db": "1.52.0" } }, "minimatch": { @@ -2813,15 +2546,6 @@ "p-limit": "^3.0.2" } }, - "p-map": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", - "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", - "dev": true, - "requires": { - "aggregate-error": "^3.0.0" - } - }, "pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", @@ -2855,24 +2579,12 @@ "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true }, - "path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true - }, "pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", "dev": true }, - "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true - }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -3011,12 +2723,6 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, - "slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true - }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -3086,15 +2792,6 @@ "rimraf": "^3.0.0" } }, - "to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "requires": { - "is-number": "^7.0.0" - } - }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 7572b75b5b4..730fe84040a 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "scripts": { "test": "teslint .", "lint": "teslint", - "lint:linter": "eslint --no-ignore .ci" + "lint:linter": "eslint --no-ignore .ci", + "updateTypes": ".ci/updateTypes.mjs" }, "author": "kba+pz", "license": "CC0-1.0", @@ -15,7 +16,7 @@ }, "devDependencies": { "@zotero/eslint-config": "^1.0.5", - "chromedriver": "^95.0.0", + "chromedriver": "^115.0.0", "eslint": "^8.38.0", "eslint-plugin-zotero-translator": "file:.ci/eslint-plugin-zotero-translator", "selenium-webdriver": "^4.0.0-alpha.7"