#!/bin/bash -eu

CITATION_INPUT_FILE="enwiki-20160305-citation-data.tql.bz2"
CITATION_RAW_TSV="citations.raw.tsv.bz2"

if [ ! -f $CITATION_RAW_TSV ]; then
    echo "Generating a TSV with CitationIRI ArticleName TemplateName PropertyName"
    bzcat $CITATION_INPUT_FILE | sed -r 's|<(.*)> <http://dbpedia.org/property/.*http://en.wikipedia.org/wiki/(.*)\?oldid=.*&template=(.*)&property=(.*)> \.|\1\t\2\t\3\t\4|g'  | bzip2 > $CITATION_RAW_TSV
fi


echo "Generating statistics for template & property occurrences" 
bzcat $CITATION_RAW_TSV | cut -f 3,4 | sort | uniq -c | sort -nr > results.template-property.count

echo "Generating statistics for property occurrences"
bzcat $CITATION_RAW_TSV | cut -f 4 | sort | uniq -c | sort -nr > results.property.count

echo "Generating statistics for template occurrences"
bzcat $CITATION_RAW_TSV | cut -f 1,3 | uniq | cut -f 2 | sort | uniq -c | sort -nr > results.template.count

echo "Generating statistics for domains referenced from citations"
bzcat $CITATION_RAW_TSV | cut -f 1 | uniq | cut -d '/' -f 3 | sed 's|^www\.||g' | sort | uniq -c | sort -nr > results.domains.count

echo "Generating statistics for domains referenced from citations"
bzcat $CITATION_RAW_TSV | cut -f 1 | sort -u | cut -d '/' -f 3 | sed 's|^www\.||g' | sort | uniq -c | sort -nr > results.domains-distinct.count

echo "Generating statistics for same citations referenced multiple times (even from same article)"
bzcat $CITATION_RAW_TSV | cut -f 1 | uniq | sort | uniq -c | grep -v -E "^\ *[0-9]{1} " | sort -nr > results.same-citations.all-articles.count
cat results.same-citations.all-articles.count | grep -v "citation.dbpedia.org" > results.same-citations.all-articles-no-hash.count

echo "Generating statistics for same citations referenced multiple times (from different articles)"
bzcat $CITATION_RAW_TSV | cut -f 1,2 | uniq | sort -u | cut -f 1 | uniq -c | grep -v -E "^\ *[0-9]{1} " | sort -nr > results.same-citations.different-articles.count
cat results.same-citations.different-articles.count | grep -v "citation.dbpedia.org" > results.same-citations.different-articles-no-hash.count

echo "Generating statistics for articles with needed citations"
bzcat $CITATION_RAW_TSV | cut -f 1,2,3 | uniq | grep "Citation_needed" | cut -f 2 | uniq -c | grep -v -E "^\ *[0-9]{1} " | sort -nr > results.articles-with-citations-neededd.count




