{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,5,4]],"date-time":"2026-05-04T00:10:55Z","timestamp":1777853455453,"version":"3.51.4"},"reference-count":33,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2020,5,1]],"date-time":"2020-05-01T00:00:00Z","timestamp":1588291200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2020,5,1]],"date-time":"2020-05-01T00:00:00Z","timestamp":1588291200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/www.elsevier.com\/legal\/tdmrep-license"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61672102"],"award-info":[{"award-number":["61672102"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61370064"],"award-info":[{"award-number":["61370064"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["60940032"],"award-info":[{"award-number":["60940032"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61073034"],"award-info":[{"award-number":["61073034"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Future Generation Computer Systems"],"published-print":{"date-parts":[[2020,5]]},"DOI":"10.1016\/j.future.2020.01.010","type":"journal-article","created":{"date-parts":[[2020,1,20]],"date-time":"2020-01-20T11:31:25Z","timestamp":1579519885000},"page":"426-437","update-policy":"https:\/\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":5,"special_numbering":"C","title":["Evaluating the crowd quality for subjective questions based on a Spark computing environment"],"prefix":"10.1016","volume":"106","author":[{"given":"Xinxin","family":"Wang","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Depeng","family":"Dang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zixian","family":"Guo","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"78","reference":[{"issue":"6","key":"10.1016\/j.future.2020.01.010_b1","first-page":"1","article-title":"The rise of crowdsourcing","volume":"14","author":"Howe","year":"2006","journal-title":"Wired Mag."},{"key":"10.1016\/j.future.2020.01.010_b2","doi-asserted-by":"crossref","unstructured":"D.C. Brabham, Crowdsourcing, in: Proceedings of the International Encyclopedia of Organizational Communication, 2017, pp. 1\u20136.","DOI":"10.1002\/9781118955567.wbieoc052"},{"issue":"2","key":"10.1016\/j.future.2020.01.010_b3","first-page":"16","article-title":"Analyzing the amazon mechanical turk marketplace. XRDS: Crossroads","volume":"17","author":"Ipeirotis","year":"2010","journal-title":"ACM Mag. Stud."},{"key":"10.1016\/j.future.2020.01.010_b4","doi-asserted-by":"crossref","unstructured":"D.E. Difallah, M. Catasta, G. Demartini, et al. The dynamics of micro-task crowdsourcing: The case of amazon mturk, in: Proceedings of the 24th International Conference on World Wide Web (WWW), 2015, pp. 238\u2013247.","DOI":"10.1145\/2736277.2741685"},{"key":"10.1016\/j.future.2020.01.010_b5","doi-asserted-by":"crossref","unstructured":"M.C. Yuen, I. King, K.S. Leung, A survey of crowdsourcing systems, in: Proceedings of the 2011 IEEE Third International Conference on Privacy, Security, Risk and Trust (PASSAT) and 2011 IEEE Third Inernational Conference on Social Computing (SocialCom), 2011, pp. 766\u2013773.","DOI":"10.1109\/PASSAT\/SocialCom.2011.203"},{"issue":"4","key":"10.1016\/j.future.2020.01.010_b6","first-page":"51","article-title":"Crowdsourcing empathetic intelligence: the case of the annotation of EMMA database for emotion and mood recognition","volume":"7","author":"Katsimerou","year":"2016","journal-title":"ACM Trans. Intell. Syst. Technol. (TIST)"},{"issue":"2","key":"10.1016\/j.future.2020.01.010_b7","doi-asserted-by":"crossref","first-page":"125","DOI":"10.14778\/2735471.2735474","article-title":"Scaling up crowd-sourcing to very large datasets: A case for active learning","volume":"8","author":"Mozafari","year":"2014","journal-title":"Proc. VLDB Endow."},{"key":"10.1016\/j.future.2020.01.010_b8","doi-asserted-by":"crossref","unstructured":"Y. Zheng, J. Wang, G. Li, R. Cheng, J. Feng, QASCA: A quality-aware task assignment system for crowdsourcing applications, in: Proceedings of the 2015 ACM SIGMOD International Conference on Management of Data, 2015, pp. 1031\u20131046.","DOI":"10.1145\/2723372.2749430"},{"key":"10.1016\/j.future.2020.01.010_b9","series-title":"Proceedings of the Extended Abstracts of the 2018 CHI Conference on Human Factors in Computing Systems","article-title":"Investigating crowdsourcing as a method to collect emotion labels for images","author":"Korovina","year":"2018"},{"key":"10.1016\/j.future.2020.01.010_b10","series-title":"Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems","first-page":"1631","article-title":"Understanding malicious behavior in crowdsourcing platforms: The case of online surveys","author":"Gadiraju","year":"2015"},{"key":"10.1016\/j.future.2020.01.010_b11","series-title":"Proceedings of the 19th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","first-page":"686","article-title":"Evaluating the crowd with confidence","author":"Joglekar","year":"2013"},{"key":"10.1016\/j.future.2020.01.010_b12","series-title":"Statistical Methods and Scientific Inference","author":"Fisher","year":"1956"},{"key":"10.1016\/j.future.2020.01.010_b13","unstructured":"C. Harris, You\u2019re hired! An examination of crowdsourcing incentive models in human resource tasks, in: Proceedings of the Workshop on Crowdsourcing for Search and Data Mining (CSDM) at the Fourth ACM International Conference on Web Search and Data Mining (WSDM), Hong Kong, China, 2011, pp. 15\u201318."},{"key":"10.1016\/j.future.2020.01.010_b14","unstructured":"D. Oleson, A. Sorokin, G. Laughlin, et al. Programmatic gold: Targeted and scalable quality assurance in crowdsourcing, in: Proceedings of the Workshops at the Twenty-Fifth AAAI Conference on Artificial Intelligence, 2011."},{"key":"10.1016\/j.future.2020.01.010_b15","series-title":"Proceedings of the Ninth ACM International Conference on Web Search and Data Mining","first-page":"267","article-title":"Quality management in crowdsourcing using gold judges behavior","author":"Kazai","year":"2016"},{"key":"10.1016\/j.future.2020.01.010_b16","series-title":"Proceedings of the International Conference on Web Information Systems Engineering","first-page":"293","article-title":"A majority of wrongs doesn\u2019t make it right-on crowdsourcing quality for skewed domain tasks","author":"El\u00a0Maarry","year":"2015"},{"key":"10.1016\/j.future.2020.01.010_b17","unstructured":"Y. Yan, R. Rosales, G. Fung, et al. Active learning from crowds, in: Proceedings of the ICML, Vol. 11, 2011, pp. 1161\u20131168."},{"key":"10.1016\/j.future.2020.01.010_b18","series-title":"Proceedings of the 2013 IEEE 33rd International Conference on Distributed Computing Systems","first-page":"530","article-title":"Recursive fact-finding: A streaming approach to truth estimation in crowdsourcing applications","author":"Wang","year":"2013"},{"issue":"1","key":"10.1016\/j.future.2020.01.010_b19","first-page":"3537","article-title":"Spectral methods meet EM: A provably optimal algorithm for crowdsourcing","volume":"17","author":"Zhang","year":"2016","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.future.2020.01.010_b20","series-title":"Proceedings of the 2013 Conference on Computer Supported Cooperative Work","first-page":"639","article-title":"Enhancing reliability using peer consistency evaluation in human computation","author":"Huang","year":"2013"},{"key":"10.1016\/j.future.2020.01.010_b21","unstructured":"N. Shah, D. Zhou, Y. Peres, Approval voting and incentives in crowdsourcing, in: Proceedings of the International Conference on Machine Learning, 2015, pp. 10\u201319."},{"issue":"99","key":"10.1016\/j.future.2020.01.010_b22","first-page":"1","article-title":"Domain-weighted majority voting for crowdsourcing","author":"Tao","year":"2018","journal-title":"IEEE Trans. Neural Netw. Learn. Syst."},{"key":"10.1016\/j.future.2020.01.010_b23","doi-asserted-by":"crossref","first-page":"29","DOI":"10.1016\/j.knosys.2017.03.003","article-title":"Victor chang towards data analysis for weather cloud computing","volume":"127","author":"Chang","year":"2017","journal-title":"Knowl.-Based Syst."},{"key":"10.1016\/j.future.2020.01.010_b24","doi-asserted-by":"crossref","first-page":"56","DOI":"10.1016\/j.future.2015.10.003","article-title":"A model to compare cloud and non-cloud storage of big data","volume":"57","author":"Chang","year":"2015","journal-title":"Future Gener. Comput. Syst."},{"key":"10.1016\/j.future.2020.01.010_b25","doi-asserted-by":"crossref","first-page":"219","DOI":"10.1016\/j.jpdc.2019.07.014","article-title":"Towards an improved adaboost algorithmic method for computational financial analysis","volume":"134","author":"Chang","year":"2019","journal-title":"J. Parallel Distrib. Comput."},{"issue":"1","key":"10.1016\/j.future.2020.01.010_b26","doi-asserted-by":"crossref","first-page":"27","DOI":"10.1007\/s10796-018-9853-8","article-title":"Towards a reuse strategic decision pattern framework\u2013from theories to practices","volume":"21","author":"Chang","year":"2019","journal-title":"Inform. Syst. Front."},{"issue":"10\u201310","key":"10.1016\/j.future.2020.01.010_b27","first-page":"95","article-title":"Spark: Cluster computing with working sets","volume":"10","author":"Zaharia","year":"2010","journal-title":"HotCloud"},{"key":"10.1016\/j.future.2020.01.010_b28","series-title":"All of Statistics: A Concise Course in Statistical Inference","author":"Wasserman","year":"2013"},{"key":"10.1016\/j.future.2020.01.010_b29","series-title":"Proceedings of the 2015 IEEE 31st International Conference on Data Engineering","first-page":"195","article-title":"Comprehensive and reliable crowd assessment algorithms","author":"Joglekar","year":"2015"},{"key":"10.1016\/j.future.2020.01.010_b30","series-title":"Learning Spark: Lightning-Fast Big Data Analysis","author":"Karau","year":"2015"},{"key":"10.1016\/j.future.2020.01.010_b31","series-title":"Proceedings of the Third Workshop on Statistical Machine Translation","first-page":"187","article-title":"The role of pseudo references in MT evaluation","author":"Albrecht","year":"2008"},{"issue":"3","key":"10.1016\/j.future.2020.01.010_b32","first-page":"377","article-title":"Algorithm AS 89: the upper tail probabilities of spearman\u2019s rho","volume":"24","author":"Best","year":"1975","journal-title":"J. R. Stat. Soc. Ser. C (Appl. Stat.)"},{"issue":"2","key":"10.1016\/j.future.2020.01.010_b33","doi-asserted-by":"crossref","first-page":"267","DOI":"10.1162\/COLI_a_00123","article-title":"Automatically assessing machine summary content without a gold standard","volume":"39","author":"Louis","year":"2013","journal-title":"Comput. Linguist."}],"container-title":["Future Generation Computer Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0167739X19313743?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/api.elsevier.com\/content\/article\/PII:S0167739X19313743?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2026,4,29]],"date-time":"2026-04-29T22:52:57Z","timestamp":1777503177000},"score":1,"resource":{"primary":{"URL":"https:\/\/linkinghub.elsevier.com\/retrieve\/pii\/S0167739X19313743"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2020,5]]},"references-count":33,"alternative-id":["S0167739X19313743"],"URL":"https:\/\/doi.org\/10.1016\/j.future.2020.01.010","relation":{},"ISSN":["0167-739X"],"issn-type":[{"value":"0167-739X","type":"print"}],"subject":[],"published":{"date-parts":[[2020,5]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Evaluating the crowd quality for subjective questions based on a Spark computing environment","name":"articletitle","label":"Article Title"},{"value":"Future Generation Computer Systems","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/doi.org\/10.1016\/j.future.2020.01.010","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2020 Elsevier B.V. All rights reserved.","name":"copyright","label":"Copyright"}]}}