Mercurial > repos > stevecassidy > nltktools

annotate g_frequency.py @ 1:fb617586f4b2 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
author stevecassidy
date Mon, 05 Dec 2016 05:22:05 -0500
parents e991d4e60c17
children a47980ef2b96
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
1 import nltk
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
2 from nltk import FreqDist
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
3 import argparse
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
4
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
5 def arguments():
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
6 parser = argparse.ArgumentParser(description="generate a word frequency table from a text")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
7 parser.add_argument('--input', required=True, action="store", type=str, help="input text file")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
8 parser.add_argument('--output', required=True, action="store", type=str, help="output file path")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
9 args = parser.parse_args()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
10 return args
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
11
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
12
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
13 def frequency(in_file, out_file):
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
14 """Input: a text file
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
15 Output: a table of word frequency with three columns for Word, Count and Percent frequency
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
16 """
1
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
17 with open(in_file, 'r') as fd:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
18 text = fd.read()
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
19
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
20 words = nltk.word_tokenize(text)
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
21 frequency = FreqDist(words)
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
22 total = float(frequency.N())
1
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
23
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
24 with open(out_file, 'w') as output:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
25 output.write("Word\tCount\tPercent\n")
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
26 for pair in frequency.items():
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
27 output.write("{pair[0]}\t{pair[1]}\t{pc:.2f}\n".format(pair=pair, pc=100*pair[1]/total))
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
28
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
29
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
30 if __name__ == '__main__':
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
31 args = arguments()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
32 frequency(args.input, args.output)