arxiv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request as libreq\n",
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "## get raw data \n",
    "## by paper  'https://api.semanticscholar.org/v1/paper/arXiv:1705.10311'\n",
    "## by author 'https://api.semanticscholar.org/v1/author/[S2AuthorId]'\n",
    "## paper by paper id  : https://api.semanticscholar.org/v1/paper/0796f6cd7f0403a854d67d525e9b32af3b277331\n",
    "b_url = 'https://api.semanticscholar.org/v1/paper/arXiv:hep-ph/0610184'\n",
    "with libreq.urlopen(b_url) as url:\n",
    "     r = url.read()\n",
    "## use https://api.semanticscholar.org/ for citation on arxiv "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "# parse x:\n",
    "data = json.loads(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['abstract', 'arxivId', 'authors', 'citationVelocity', 'citations', 'corpusId', 'doi', 'fieldsOfStudy', 'influentialCitationCount', 'is_open_access', 'is_publisher_licensed', 'paperId', 'references', 'title', 'topics', 'url', 'venue', 'year'])"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Three Graded Modified Classical Yang-Baxter Equations and Integrable Systems'"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'arxivId': None,\n",
       " 'authors': [{'authorId': '150265684', 'name': 'Pradyumna Prakhar Sinha'},\n",
       "  {'authorId': '9379166', 'name': 'Rohan Mishra'},\n",
       "  {'authorId': '51042088', 'name': 'Ramit Sawhney'},\n",
       "  {'authorId': '2712224', 'name': 'Debanjan Mahata'},\n",
       "  {'authorId': '1753278', 'name': 'R. Shah'},\n",
       "  {'authorId': '145896397', 'name': 'Huan Liu'}],\n",
       " 'doi': '10.1145/3357384.3358060',\n",
       " 'intent': ['methodology'],\n",
       " 'isInfluential': False,\n",
       " 'paperId': '3a14976209525f3a812dd49345d9c553df62f5ff',\n",
       " 'title': '#suicidal - A Multipronged Approach to Identify and Explore Suicidal Ideation in Twitter',\n",
       " 'url': 'https://www.semanticscholar.org/paper/3a14976209525f3a812dd49345d9c553df62f5ff',\n",
       " 'venue': 'CIKM',\n",
       " 'year': 2019}"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['citations'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'topic': 'Simulation',\n",
       "  'topicId': '194',\n",
       "  'url': 'https://www.semanticscholar.org/topic/194'},\n",
       " {'topic': 'Robotics',\n",
       "  'topicId': '2759',\n",
       "  'url': 'https://www.semanticscholar.org/topic/2759'},\n",
       " {'topic': 'Reinforcement learning',\n",
       "  'topicId': '2557',\n",
       "  'url': 'https://www.semanticscholar.org/topic/2557'},\n",
       " {'topic': 'Real life',\n",
       "  'topicId': '9808',\n",
       "  'url': 'https://www.semanticscholar.org/topic/9808'},\n",
       " {'topic': 'World-system',\n",
       "  'topicId': '495901',\n",
       "  'url': 'https://www.semanticscholar.org/topic/495901'},\n",
       " {'topic': 'Experiment',\n",
       "  'topicId': '378',\n",
       "  'url': 'https://www.semanticscholar.org/topic/378'},\n",
       " {'topic': 'Algorithm',\n",
       "  'topicId': '305',\n",
       "  'url': 'https://www.semanticscholar.org/topic/305'},\n",
       " {'topic': 'Sparse matrix',\n",
       "  'topicId': '126',\n",
       "  'url': 'https://www.semanticscholar.org/topic/126'},\n",
       " {'topic': 'Robot',\n",
       "  'topicId': '6657',\n",
       "  'url': 'https://www.semanticscholar.org/topic/6657'}]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['topics']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'arxivId': None,\n",
       "  'authors': [{'authorId': '81555516', 'name': 'Fabio Muratore'},\n",
       "   {'authorId': '80318089', 'name': 'Felix Treede'},\n",
       "   {'authorId': '1713430', 'name': 'M. Gienger'},\n",
       "   {'authorId': '145197867', 'name': 'Jan Peters'}],\n",
       "  'doi': None,\n",
       "  'intent': ['methodology', 'background'],\n",
       "  'isInfluential': False,\n",
       "  'paperId': '4ec94d6968bcb0a030a658d7347ce15e64588528',\n",
       "  'title': 'Domain Randomization for Simulation-Based Policy Optimization with Transferability Assessment',\n",
       "  'url': 'https://www.semanticscholar.org/paper/4ec94d6968bcb0a030a658d7347ce15e64588528',\n",
       "  'venue': 'CoRL',\n",
       "  'year': 2018},\n",
       " {'arxivId': '1907.04685',\n",
       "  'authors': [{'authorId': '81555516', 'name': 'Fabio Muratore'},\n",
       "   {'authorId': '1713430', 'name': 'M. Gienger'},\n",
       "   {'authorId': '145197867', 'name': 'Jan Peters'}],\n",
       "  'doi': '10.1109/tpami.2019.2952353',\n",
       "  'intent': ['methodology', 'background'],\n",
       "  'isInfluential': False,\n",
       "  'paperId': '1a20a92125904772261fdf646881121059932ce6',\n",
       "  'title': 'Assessing Transferability from Simulation to Reality for Reinforcement Learning',\n",
       "  'url': 'https://www.semanticscholar.org/paper/1a20a92125904772261fdf646881121059932ce6',\n",
       "  'venue': 'IEEE transactions on pattern analysis and machine intelligence',\n",
       "  'year': 2019}]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['citations']   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cs.LG'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "soup = BeautifulSoup(r)\n",
    "## get all papers\n",
    "entries = soup.find_all('entry')\n",
    "## get paper title (as example)\n",
    "entries[0].title\n",
    "## get atributes of a class (category as example )\n",
    "## VISIT \"https://arxiv.org/help/prep#subj\" for category term meaning \n",
    "cats = entries[0].find_all('category')\n",
    "cats[0].get('term')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<arxiv:primary_category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.LG\" xmlns:arxiv=\"http://arxiv.org/schemas/atom\"></arxiv:primary_category>]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entries[0].find_all('arxiv:primary_category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<entry>\n",
       "<id>http://arxiv.org/abs/1707.09562v3</id>\n",
       "<updated>2017-10-16T11:13:32Z</updated>\n",
       "<published>2017-07-29T21:59:18Z</published>\n",
       "<title>MLBench: How Good Are Machine Learning Clouds for Binary Classification\n",
       "  Tasks on Structured Data?</title>\n",
       "<summary>  We conduct an empirical study of machine learning functionalities provided by\n",
       "major cloud service providers, which we call machine learning clouds. Machine\n",
       "learning clouds hold the promise of hiding all the sophistication of running\n",
       "large-scale machine learning: Instead of specifying how to run a machine\n",
       "learning task, users only specify what machine learning task to run and the\n",
       "cloud figures out the rest. Raising the level of abstraction, however, rarely\n",
       "comes free - a performance penalty is possible. How good, then, are current\n",
       "machine learning clouds on real-world machine learning workloads?\n",
       "  We study this question with a focus on binary classication problems. We\n",
       "present mlbench, a novel benchmark constructed by harvesting datasets from\n",
       "Kaggle competitions. We then compare the performance of the top winning code\n",
       "available from Kaggle with that of running machine learning clouds from both\n",
       "Azure and Amazon on mlbench. Our comparative study reveals the strength and\n",
       "weakness of existing machine learning clouds and points out potential future\n",
       "directions for improvement.\n",
       "</summary>\n",
       "<author>\n",
       "<name>Yu Liu</name>\n",
       "</author>\n",
       "<author>\n",
       "<name>Hantian Zhang</name>\n",
       "</author>\n",
       "<author>\n",
       "<name>Luyuan Zeng</name>\n",
       "</author>\n",
       "<author>\n",
       "<name>Wentao Wu</name>\n",
       "</author>\n",
       "<author>\n",
       "<name>Ce Zhang</name>\n",
       "</author>\n",
       "<link href=\"http://arxiv.org/abs/1707.09562v3\" rel=\"alternate\" type=\"text/html\"/>\n",
       "<link href=\"http://arxiv.org/pdf/1707.09562v3\" rel=\"related\" title=\"pdf\" type=\"application/pdf\"/>\n",
       "<arxiv:primary_category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.DC\" xmlns:arxiv=\"http://arxiv.org/schemas/atom\"></arxiv:primary_category>\n",
       "<category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.DC\"></category>\n",
       "<category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.LG\"></category>\n",
       "<category scheme=\"http://arxiv.org/schemas/atom\" term=\"stat.ML\"></category>\n",
       "</entry>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entries[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://export.arxiv.org/api/query?search_query=all:deeplearning&start=0&max_results=5&sortBy=relevance&sortOrder=descending\n",
      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
      "<html>\n",
      " <body>\n",
      "  <feed xmlns=\"http://www.w3.org/2005/Atom\">\n",
      "   <link href=\"http://arxiv.org/api/query?search_query%3Dall%3Adeeplearning%26id_list%3D%26start%3D0%26max_results%3D5\" rel=\"self\" type=\"application/atom+xml\"/>\n",
      "   <title type=\"html\">\n",
      "    ArXiv Query: search_query=all:deeplearning&amp;id_list=&amp;start=0&amp;max_results=5\n",
      "   </title>\n",
      "   <id>\n",
      "    http://arxiv.org/api/twn3ljWGNLphYUOhZcKii3R2T8I\n",
      "   </id>\n",
      "   <updated>\n",
      "    2020-12-22T00:00:00-05:00\n",
      "   </updated>\n",
      "   <opensearch:totalresults xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\">\n",
      "    8\n",
      "   </opensearch:totalresults>\n",
      "   <opensearch:startindex xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\">\n",
      "    0\n",
      "   </opensearch:startindex>\n",
      "   <opensearch:itemsperpage xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\">\n",
      "    5\n",
      "   </opensearch:itemsperpage>\n",
      "   <entry>\n",
      "    <id>\n",
      "     http://arxiv.org/abs/1611.01884v3\n",
      "    </id>\n",
      "    <updated>\n",
      "     2017-06-05T03:47:15Z\n",
      "    </updated>\n",
      "    <published>\n",
      "     2016-11-07T03:39:52Z\n",
      "    </published>\n",
      "    <title>\n",
      "     AC-BLSTM: Asymmetric Convolutional Bidirectional LSTM Networks for Text\n",
      "  Classification\n",
      "    </title>\n",
      "    <summary>\n",
      "     Recently deeplearning models have been shown to be capable of making\n",
      "remarkable performance in sentences and documents classification tasks. In this\n",
      "work, we propose a novel framework called AC-BLSTM for modeling sentences and\n",
      "documents, which combines the asymmetric convolution neural network (ACNN) with\n",
      "the Bidirectional Long Short-Term Memory network (BLSTM). Experiment results\n",
      "demonstrate that our model achieves state-of-the-art results on five tasks,\n",
      "including sentiment analysis, question type classification, and subjectivity\n",
      "classification. In order to further improve the performance of AC-BLSTM, we\n",
      "propose a semi-supervised learning framework called G-AC-BLSTM for text\n",
      "classification by combining the generative model with AC-BLSTM.\n",
      "    </summary>\n",
      "    <author>\n",
      "     <name>\n",
      "      Depeng Liang\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Yongdong Zhang\n",
      "     </name>\n",
      "    </author>\n",
      "    <arxiv:comment xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "     9 pages\n",
      "    </arxiv:comment>\n",
      "    <link href=\"http://arxiv.org/abs/1611.01884v3\" rel=\"alternate\" type=\"text/html\"/>\n",
      "    <link href=\"http://arxiv.org/pdf/1611.01884v3\" rel=\"related\" title=\"pdf\" type=\"application/pdf\"/>\n",
      "    <arxiv:primary_category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.CL\" xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "    </arxiv:primary_category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.CL\">\n",
      "    </category>\n",
      "   </entry>\n",
      "   <entry>\n",
      "    <id>\n",
      "     http://arxiv.org/abs/1907.08356v1\n",
      "    </id>\n",
      "    <updated>\n",
      "     2019-07-19T03:04:50Z\n",
      "    </updated>\n",
      "    <published>\n",
      "     2019-07-19T03:04:50Z\n",
      "    </published>\n",
      "    <title>\n",
      "     New Era of Deeplearning-Based Malware Intrusion Detection: The Malware\n",
      "  Detection and Prediction Based On Deep Learning\n",
      "    </title>\n",
      "    <summary>\n",
      "     With the development of artificial intelligence algorithms like deep learning\n",
      "models and the successful applications in many different fields, further\n",
      "similar trails of deep learning technology have been made in cyber security\n",
      "area. It shows the preferable performance not only in academic security\n",
      "research but also in industry practices when dealing with part of cyber\n",
      "security issues by deep learning methods compared to those conventional rules.\n",
      "Especially for the malware detection and classification tasks, it saves\n",
      "generous time cost and promotes the accuracy for a total pipeline of malware\n",
      "detection system. In this paper, we construct special deep neural network, ie,\n",
      "MalDeepNet (TB-Malnet and IB-Malnet) for malware dynamic behavior\n",
      "classification tasks. Then we build the family clustering algorithm based on\n",
      "deep learning and fulfil related testing. Except that, we also design a novel\n",
      "malware prediction model which could detect the malware coming in future\n",
      "through the Mal Generative Adversarial Network (Mal-GAN) implementation. All\n",
      "those algorithms present fairly considerable value in related datasets\n",
      "afterwards.\n",
      "    </summary>\n",
      "    <author>\n",
      "     <name>\n",
      "      Shuqiang Lu\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Lingyun Ying\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Wenjie Lin\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Yu Wang\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Meining Nie\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Kaiwen Shen\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Lu Liu\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Haixin Duan\n",
      "     </name>\n",
      "    </author>\n",
      "    <link href=\"http://arxiv.org/abs/1907.08356v1\" rel=\"alternate\" type=\"text/html\"/>\n",
      "    <link href=\"http://arxiv.org/pdf/1907.08356v1\" rel=\"related\" title=\"pdf\" type=\"application/pdf\"/>\n",
      "    <arxiv:primary_category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.CR\" xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "    </arxiv:primary_category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.CR\">\n",
      "    </category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.LG\">\n",
      "    </category>\n",
      "   </entry>\n",
      "   <entry>\n",
      "    <id>\n",
      "     http://arxiv.org/abs/2003.00803v1\n",
      "    </id>\n",
      "    <updated>\n",
      "     2020-02-09T20:23:08Z\n",
      "    </updated>\n",
      "    <published>\n",
      "     2020-02-09T20:23:08Z\n",
      "    </published>\n",
      "    <title>\n",
      "     Ascertaining price formation in cryptocurrency markets with DeepLearning\n",
      "    </title>\n",
      "    <summary>\n",
      "     The cryptocurrency market is amongst the fastest-growing of all the financial\n",
      "markets in the world. Unlike traditional markets, such as equities, foreign\n",
      "exchange and commodities, cryptocurrency market is considered to have larger\n",
      "volatility and illiquidity. This paper is inspired by the recent success of\n",
      "using deep learning for stock market prediction. In this work, we analyze and\n",
      "present the characteristics of the cryptocurrency market in a high-frequency\n",
      "setting. In particular, we applied a deep learning approach to predict the\n",
      "direction of the mid-price changes on the upcoming tick. We monitored live\n",
      "tick-level data from $8$ cryptocurrency pairs and applied both statistical and\n",
      "machine learning techniques to provide a live prediction. We reveal that\n",
      "promising results are possible for cryptocurrencies, and in particular, we\n",
      "achieve a consistent $78\\%$ accuracy on the prediction of the mid-price\n",
      "movement on live exchange rate of Bitcoins vs US dollars.\n",
      "    </summary>\n",
      "    <author>\n",
      "     <name>\n",
      "      Fan Fang\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Waichung Chung\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Carmine Ventre\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Michail Basios\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Leslie Kanthan\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Lingbo Li\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Fan Wu\n",
      "     </name>\n",
      "    </author>\n",
      "    <link href=\"http://arxiv.org/abs/2003.00803v1\" rel=\"alternate\" type=\"text/html\"/>\n",
      "    <link href=\"http://arxiv.org/pdf/2003.00803v1\" rel=\"related\" title=\"pdf\" type=\"application/pdf\"/>\n",
      "    <arxiv:primary_category scheme=\"http://arxiv.org/schemas/atom\" term=\"q-fin.GN\" xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "    </arxiv:primary_category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"q-fin.GN\">\n",
      "    </category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.LG\">\n",
      "    </category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"q-fin.TR\">\n",
      "    </category>\n",
      "   </entry>\n",
      "   <entry>\n",
      "    <id>\n",
      "     http://arxiv.org/abs/2004.14178v1\n",
      "    </id>\n",
      "    <updated>\n",
      "     2020-04-29T13:21:28Z\n",
      "    </updated>\n",
      "    <published>\n",
      "     2020-04-29T13:21:28Z\n",
      "    </published>\n",
      "    <title>\n",
      "     Deepfake Video Forensics based on Transfer Learning\n",
      "    </title>\n",
      "    <summary>\n",
      "     Deeplearning has been used to solve complex problems in various domains. As\n",
      "it advances, it also creates applications which become a major threat to our\n",
      "privacy, security and even to our Democracy. Such an application which is being\n",
      "developed recently is the \"Deepfake\". Deepfake models can create fake images\n",
      "and videos that humans cannot differentiate them from the genuine ones.\n",
      "Therefore, the counter application to automatically detect and analyze the\n",
      "digital visual media is necessary in today world. This paper details retraining\n",
      "the image classification models to apprehend the features from each deepfake\n",
      "video frames. After feeding different sets of deepfake clips of video fringes\n",
      "through a pretrained layer of bottleneck in the neural network is made for\n",
      "every video frame, already stated layer contains condense data for all images\n",
      "and exposes artificial manipulations in Deepfake videos. When checking Deepfake\n",
      "videos, this technique received more than 87 per cent accuracy. This technique\n",
      "has been tested on the Face Forensics dataset and obtained good accuracy in\n",
      "detection.\n",
      "    </summary>\n",
      "    <author>\n",
      "     <name>\n",
      "      Rahul U\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Ragul M\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Raja Vignesh K\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Tejeswinee K\n",
      "     </name>\n",
      "    </author>\n",
      "    <arxiv:doi xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "     10.35940/ijrte.F9747.038620\n",
      "    </arxiv:doi>\n",
      "    <link href=\"http://dx.doi.org/10.35940/ijrte.F9747.038620\" rel=\"related\" title=\"doi\"/>\n",
      "    <arxiv:comment xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "     This submission has been removed by arXiv administrators due to\n",
      "  copyright infringement\n",
      "    </arxiv:comment>\n",
      "    <link href=\"http://arxiv.org/abs/2004.14178v1\" rel=\"alternate\" type=\"text/html\"/>\n",
      "    <link href=\"http://arxiv.org/pdf/2004.14178v1\" rel=\"related\" title=\"pdf\" type=\"application/pdf\"/>\n",
      "    <arxiv:primary_category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.CV\" xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "    </arxiv:primary_category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.CV\">\n",
      "    </category>\n",
      "   </entry>\n",
      "   <entry>\n",
      "    <id>\n",
      "     http://arxiv.org/abs/1903.06254v1\n",
      "    </id>\n",
      "    <updated>\n",
      "     2019-03-11T13:01:38Z\n",
      "    </updated>\n",
      "    <published>\n",
      "     2019-03-11T13:01:38Z\n",
      "    </published>\n",
      "    <title>\n",
      "     Demonstration of Vector Flow Imaging using Convolutional Neural Networks\n",
      "    </title>\n",
      "    <summary>\n",
      "     Synthetic Aperture Vector Flow Imaging (SA-VFI) can visualize complex cardiac\n",
      "and vascular blood flow patterns at high temporal resolution with a large field\n",
      "of view. Convolutional neural networks (CNNs) are commonly used in image and\n",
      "video recognition and classification. However, most recently presented CNNs\n",
      "also allow for making per-pixel predictions as needed in optical flow\n",
      "velocimetry. To our knowledge we demonstrate here for the first time a CNN\n",
      "architecture to produce 2D full flow field predictions from high frame rate SA\n",
      "ultrasound images using supervised learning. The CNN was initially trained\n",
      "using CFD-generated and augmented noiseless SA ultrasound data of a realistic\n",
      "vessel geometry. Subsequently, a mix of noisy simulated and real \\textit{in\n",
      "vivo} acquisitions were added to increase the network's robustness. The\n",
      "resulting flow field of the CNN resembled the ground truth accurately with an\n",
      "endpoint-error percentage between 6.5\\% to 14.5\\%. Furthermore, when confronted\n",
      "with an unknown geometry of an arterial bifurcation, the CNN was able to\n",
      "predict an accurate flow field indicating its ability for generalization.\n",
      "Remarkably, the CNN also performed well for rotational flows, which usually\n",
      "requires advanced, computationally intensive VFI methods. We have demonstrated\n",
      "that convolutional neural networks can be used to estimate complex\n",
      "multidirectional flow.\n",
      "    </summary>\n",
      "    <author>\n",
      "     <name>\n",
      "      Thomas Robins\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Antonio Stanziola\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Kai Reimer\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Peter Weinberg\n",
      "     </name>\n",
      "    </author>\n",
      "    <author>\n",
      "     <name>\n",
      "      Meng-Xing Tang\n",
      "     </name>\n",
      "    </author>\n",
      "    <arxiv:comment xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "     2018 IEEE International Ultrasonics Symposium, Convolutional Neural\n",
      "  Network, DeepLearning, Echo-PIV, UltrasoundToolbox, FieldII\n",
      "    </arxiv:comment>\n",
      "    <link href=\"http://arxiv.org/abs/1903.06254v1\" rel=\"alternate\" type=\"text/html\"/>\n",
      "    <link href=\"http://arxiv.org/pdf/1903.06254v1\" rel=\"related\" title=\"pdf\" type=\"application/pdf\"/>\n",
      "    <arxiv:primary_category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.CV\" xmlns:arxiv=\"http://arxiv.org/schemas/atom\">\n",
      "    </arxiv:primary_category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"cs.CV\">\n",
      "    </category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"eess.IV\">\n",
      "    </category>\n",
      "    <category scheme=\"http://arxiv.org/schemas/atom\" term=\"eess.SP\">\n",
      "    </category>\n",
      "   </entry>\n",
      "  </feed>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "import urllib\n",
    "import feedparser\n",
    "\n",
    "# Base api query url\n",
    "base_url = 'http://export.arxiv.org/api/query?';\n",
    "\n",
    "# Search parameters\n",
    "#  to join queries : 'cat:cs.LG+AND+ti:deeplearning'\n",
    "search_query = 'all:deeplearning' # search for electron in all fields\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "start = 0                     # retreive the first 5 results\n",
    "max_results = 5\n",
    "\n",
    "query = 'search_query=%s&start=%i&max_results=%i&sortBy=relevance&sortOrder=descending' % (search_query,\n",
    "                                                     start,\n",
    "                                                     max_results)\n",
    "print(base_url+query)\n",
    "with libreq.urlopen(base_url+query) as url:\n",
    "     r = url.read()\n",
    "\n",
    "soup = BeautifulSoup(r)\n",
    "print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'8'"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find('opensearch:totalresults').get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cs.LG'"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "## get all papers\n",
    "entries = soup.find_all('entry')\n",
    "## get paper title (as example)\n",
    "entries[0].title\n",
    "## get atributes of a class (category as example )\n",
    "## VISIT \"https://arxiv.org/help/prep#subj\" for category term meaning \n",
    "cats = entries[0].find_all('category')\n",
    "cats[0].get('term')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "114"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## what can we do with arxiv api \n",
    "## get data by keyword \n",
    "## custom query by  cat , title , abstract , journal + you can join multiple cats \n",
    "## get total result count by opensearch:totalresults tag \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Arxiv parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## utils\n",
    "import urllib.request as libreq\n",
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "\n",
    "def get_arxiv_papers(keyword ,sortBy='relevance' , start = 0 ,max_results=10 ):\n",
    "    base_url = 'http://export.arxiv.org/api/query?'\n",
    "    \n",
    "    search_query = 'all:'+keyword\n",
    "                 \n",
    "    query = 'search_query=%s&start=%i&max_results=%i&sortBy=relevance&sortOrder=descending' % (search_query,\n",
    "                                                     start,\n",
    "                                                     max_results)\n",
    "    \n",
    "    \n",
    "    with libreq.urlopen(base_url+query) as url:\n",
    "        r = url.read()\n",
    "    soup = BeautifulSoup(r)\n",
    "    \n",
    "    ## retrive return resp attr needed \n",
    "    \n",
    "    ## query total entries \n",
    "    total_entries_count = int(soup.find('opensearch:totalresults').get_text())\n",
    "    \n",
    "    ## query entries \n",
    "    entries = soup.find_all('entry')\n",
    "    return total_entries_count,[{\"arxiv_id\":entry.find('id').get_text().split('abs/')[-1][:-2] , \n",
    "                                 \"title\":entry.find('title').get_text() ,\n",
    "                                 \"published_at\" : entry.find('published').get_text() , \n",
    "                                 'updated_at' : entry.find('updated').get_text() , \n",
    "     } for entry in entries]\n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'arxiv_id': '1911.01914', 'title': 'A Comparative Analysis of XGBoost', 'published_at': '2019-11-05T16:18:29Z', 'updated_at': '2019-11-05T16:18:29Z'}, {'arxiv_id': '2007.05758', 'title': 'Feature Interactions in XGBoost', 'published_at': '2020-07-11T12:17:40Z', 'updated_at': '2020-07-11T12:17:40Z'}, {'arxiv_id': '1710.03634', 'title': 'LinXGBoost: Extension of XGBoost to Generalized Local Linear Models', 'published_at': '2017-10-10T14:52:47Z', 'updated_at': '2017-10-10T14:52:47Z'}, {'arxiv_id': '1806.11248', 'title': 'XGBoost: Scalable GPU Accelerated Learning', 'published_at': '2018-06-29T02:05:32Z', 'updated_at': '2018-06-29T02:05:32Z'}, {'arxiv_id': '1901.08433', 'title': 'A XGBoost risk model via feature selection and Bayesian hyper-parameter\\n  optimization', 'published_at': '2019-01-24T14:40:33Z', 'updated_at': '2019-01-24T14:40:33Z'}, {'arxiv_id': '1908.01672', 'title': 'Imbalance-XGBoost: Leveraging Weighted and Focal Losses for Binary\\n  Label-Imbalanced Classification with XGBoost', 'published_at': '2019-08-05T15:01:28Z', 'updated_at': '2019-08-05T15:01:28Z'}, {'arxiv_id': '2009.01442', 'title': 'FairXGBoost: Fairness-aware Classification in XGBoost', 'published_at': '2020-09-03T04:08:23Z', 'updated_at': '2020-10-07T05:14:38Z'}, {'arxiv_id': '2006.04920', 'title': 'Survival regression with accelerated failure time model in XGBoost', 'published_at': '2020-06-08T20:34:20Z', 'updated_at': '2020-06-11T20:04:08Z'}, {'arxiv_id': '2010.02524', 'title': 'Secure Collaborative Training and Inference for XGBoost', 'published_at': '2020-10-06T06:56:05Z', 'updated_at': '2020-10-06T06:56:05Z'}, {'arxiv_id': '1603.02754', 'title': 'XGBoost: A Scalable Tree Boosting System', 'published_at': '2016-03-09T01:11:51Z', 'updated_at': '2016-06-10T23:23:51Z'}]\n"
     ]
    }
   ],
   "source": [
    "## example\n",
    "(max_entries,entries) = get_arxiv_papers(keyword='xgboost' ,sortBy='relevance' ,start = 0 , max_results=10 )    \n",
    "print(entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def semantic_scholar_paper(paper_id , is_external =False ,citationType='relevance') : \n",
    "    \n",
    "    b_url_from_arxiv = 'https://api.semanticscholar.org/v1/paper/arXiv:'#'hep-ph/0610184'\n",
    "    b_url = 'https://api.semanticscholar.org/v1/paper/'#'0796f6cd7f0403a854d67d525e9b32af3b277331'\n",
    "\n",
    "\n",
    "    final_url = is_external*(b_url_from_arxiv+paper_id) + (not is_external )*(b_url+paper_id)\n",
    "    print(final_url)\n",
    "    with libreq.urlopen(final_url) as url:\n",
    "         r = url.read()\n",
    "    data = json.loads(r)\n",
    "    #print(data)\n",
    "    return data\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://api.semanticscholar.org/v1/paper/arXiv:1911.01914\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'libreq' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-c7b2c83e2cfa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msemantic_scholar_paper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpaper_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'1911.01914'\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mis_external\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-2-92f2828d1921>\u001b[0m in \u001b[0;36msemantic_scholar_paper\u001b[0;34m(paper_id, is_external, citationType)\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mfinal_url\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mis_external\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb_url_from_arxiv\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mpaper_id\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mnot\u001b[0m \u001b[0mis_external\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb_url\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mpaper_id\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfinal_url\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0mlibreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0murlopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfinal_url\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m          \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'libreq' is not defined"
     ]
    }
   ],
   "source": [
    "semantic_scholar_paper(paper_id='1911.01914' , is_external = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://api.semanticscholar.org/v1/paper/0796f6cd7f0403a854d67d525e9b32af3b277331\n"
     ]
    }
   ],
   "source": [
    "\n",
    "data = semantic_scholar_paper(paper_id='0796f6cd7f0403a854d67d525e9b32af3b277331' ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['abstract', 'arxivId', 'authors', 'citationVelocity', 'citations', 'corpusId', 'doi', 'fieldsOfStudy', 'influentialCitationCount', 'is_open_access', 'is_publisher_licensed', 'paperId', 'references', 'title', 'topics', 'url', 'venue', 'year'])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'arxivId': None,\n",
       " 'authors': [{'authorId': '2339397', 'name': 'M. Banko'},\n",
       "  {'authorId': '1741101', 'name': 'Oren Etzioni'}],\n",
       " 'doi': None,\n",
       " 'intent': ['background'],\n",
       " 'isInfluential': False,\n",
       " 'paperId': '6c8898cda9a1f13607e24306f6f64f20e0ff2ae7',\n",
       " 'title': 'The Tradeoffs Between Open and Traditional Relation Extraction',\n",
       " 'url': 'https://www.semanticscholar.org/paper/6c8898cda9a1f13607e24306f6f64f20e0ff2ae7',\n",
       " 'venue': 'ACL',\n",
       " 'year': 2008}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['references'][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "too many values to unpack (expected 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-2f38bbdbec72>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0minfluentialCitationCount\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: too many values to unpack (expected 2)"
     ]
    }
   ],
   "source": [
    "(title,influentialCitationCount) = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'arxivId': None,\n",
       " 'authors': [{'authorId': None, 'name': 'XU Zeng-lin'},\n",
       "  {'authorId': None, 'name': 'SHENG Yong-pan'},\n",
       "  {'authorId': '71073845', 'name': 'He Li-rong'},\n",
       "  {'authorId': '49619748', 'name': 'Wang Ya-fang'}],\n",
       " 'doi': None,\n",
       " 'intent': [],\n",
       " 'isInfluential': False,\n",
       " 'paperId': '34263fd001524c342b74911ce22f8b78598a9288',\n",
       " 'title': 'Review on Knowledge Graph Techniques',\n",
       " 'url': 'https://www.semanticscholar.org/paper/34263fd001524c342b74911ce22f8b78598a9288',\n",
       " 'venue': '',\n",
       " 'year': 2016}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['citations'][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'arxivId': None,\n",
       " 'authors': [{'authorId': '47120191', 'name': 'Xiaolan Wang'},\n",
       "  {'authorId': '48477264', 'name': 'X. Dong'},\n",
       "  {'authorId': '48513301', 'name': 'Y. Li'},\n",
       "  {'authorId': '2283085', 'name': 'A. Meliou'}],\n",
       " 'doi': '10.1109/ICDE.2019.00058',\n",
       " 'intent': ['methodology', 'background'],\n",
       " 'isInfluential': False,\n",
       " 'paperId': 'a3c4b2a300ab919bbc27df2da9ced9df1c6e8567',\n",
       " 'title': 'MIDAS: Finding the Right Web Sources to Fill Knowledge Gaps',\n",
       " 'url': 'https://www.semanticscholar.org/paper/a3c4b2a300ab919bbc27df2da9ced9df1c6e8567',\n",
       " 'venue': '2019 IEEE 35th International Conference on Data Engineering (ICDE)',\n",
       " 'year': 2019}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['citations'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'arxivId': None,\n",
       " 'authors': [{'authorId': '1691993', 'name': 'E. Riloff'}],\n",
       " 'doi': None,\n",
       " 'intent': ['background'],\n",
       " 'isInfluential': False,\n",
       " 'paperId': 'acec622ca4fb7e01a56116522d35ded149969d0a',\n",
       " 'title': 'Automatically Generating Extraction Patterns from Untagged Text',\n",
       " 'url': 'https://www.semanticscholar.org/paper/acec622ca4fb7e01a56116522d35ded149969d0a',\n",
       " 'venue': 'AAAI/IAAI, Vol. 2',\n",
       " 'year': 1996}"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['references'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "185"
      ]
     },
     "execution_count": 255,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['influentialCitationCount']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-c5d84736ba45>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'data' is not defined"
     ]
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sub_graph_from(paper_id,neighbors_tresh=3):\n",
    "    raw_data = semantic_scholar_paper(paper_id=paper_id)\n",
    "    ## get citations and refs \n",
    "    citations = raw_data.get('citations' , False) \n",
    "    refs = raw_data.get('references' , False) \n",
    "    print(len(citations) , len(refs))\n",
    "    ## process citations and refs to new_nodes/links\n",
    "    new_nodes = [{\"id\":citation.get('paperId')} for citation in citations][:neighbors_tresh] + [{\"id\":ref.get('paperId') } for ref in refs][:neighbors_tresh]\n",
    "    new_links = [{\"target\":citation.get('paperId') , \"source\":paper_id} for citation in citations][:neighbors_tresh] + [{\"target\":ref.get('paperId') , \"source\":paper_id}  for ref in refs][:neighbors_tresh]\n",
    "    print(len(new_nodes),len(new_links))\n",
    "    return new_nodes ,new_links\n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://api.semanticscholar.org/v1/paper/0796f6cd7f0403a854d67d525e9b32af3b277331\n",
      "1124 31\n",
      "6 6\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([{'id': 'a3c4b2a300ab919bbc27df2da9ced9df1c6e8567'},\n",
       "  {'id': '004746a32342fe2b523a3d2a6b27f37837239978'},\n",
       "  {'id': 'dd555c20c59bb34a404272d116b866a58b9a5f15'},\n",
       "  {'id': '04f212baaa8611eeb750e6cdc0dae71c0f5a5e86'},\n",
       "  {'id': 'b71c8e582e8e56d839c962b923b0b79bada2a7f8'},\n",
       "  {'id': 'cf53bda1fbaf6a70da4dd541423caab72267cf47'}],\n",
       " [{'target': 'a3c4b2a300ab919bbc27df2da9ced9df1c6e8567',\n",
       "   'source': '0796f6cd7f0403a854d67d525e9b32af3b277331'},\n",
       "  {'target': '004746a32342fe2b523a3d2a6b27f37837239978',\n",
       "   'source': '0796f6cd7f0403a854d67d525e9b32af3b277331'},\n",
       "  {'target': 'dd555c20c59bb34a404272d116b866a58b9a5f15',\n",
       "   'source': '0796f6cd7f0403a854d67d525e9b32af3b277331'},\n",
       "  {'target': '04f212baaa8611eeb750e6cdc0dae71c0f5a5e86',\n",
       "   'source': '0796f6cd7f0403a854d67d525e9b32af3b277331'},\n",
       "  {'target': 'b71c8e582e8e56d839c962b923b0b79bada2a7f8',\n",
       "   'source': '0796f6cd7f0403a854d67d525e9b32af3b277331'},\n",
       "  {'target': 'cf53bda1fbaf6a70da4dd541423caab72267cf47',\n",
       "   'source': '0796f6cd7f0403a854d67d525e9b32af3b277331'}])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_graph_from('0796f6cd7f0403a854d67d525e9b32af3b277331')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_graph_from('0796f6cd7f0403a854d67d525e9b32af3b277331')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## utils for data model of shemantic \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## long term Todos \n",
    "## use scholar lib with proxy tor \n",
    "## parse shemanticscholar  with keyword insted of arxiv "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}