Mercurial > repos > stevecassidy > nltktools

annotate g_tokenize.py @ 1:fb617586f4b2 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
author stevecassidy
date Mon, 05 Dec 2016 05:22:05 -0500
parents e991d4e60c17
children a47980ef2b96
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
1 import nltk
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
2 import string
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
3 import argparse
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
4
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
5 def arguments():
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
6 parser = argparse.ArgumentParser(description="tokenize a text")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
7 parser.add_argument('--input', required=True, action="store", type=str, help="input text file")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
8 parser.add_argument('--output', required=True, action="store", type=str, help="output file path")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
9 parser.add_argument('--lower', required=False, action="store_true", help="lowercase all words")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
10 parser.add_argument('--nopunct', required=False, action="store_true", help="remove all punctuation characters")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
11 args = parser.parse_args()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
12 return args
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
13
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
14
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
15 def strip_punct(text):
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
16 table = string.maketrans("","")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
17 return text.translate(table, string.punctuation)
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
18
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
19
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
20 def tokenize(in_file, out_file, lower=False, nopunct=False):
1
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
21 with open(in_file, 'r') as fd:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
22 text = fd.read()
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
23
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
24 if lower:
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
25 text = text.lower()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
26 if nopunct:
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
27 text = strip_punct(text)
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
28 result = []
1
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
29 #text = unicode(text, errors='ignore')
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
30 sentences = nltk.sent_tokenize(text)
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
31 for sentence in sentences:
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
32 tokens = nltk.word_tokenize(sentence)
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
33 result.append(tokens)
1
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
34 with open(out_file, 'w') as output:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
35 # write one token per line
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
36 for sentence in result:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
37 for token in sentence:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
38 output.write(token + "\n")
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
39
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
40
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
41 if __name__ == '__main__':
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
42 args = arguments()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
43 tokenize(args.input, args.output, lower=args.lower, nopunct=args.nopunct)