Mercurial > repos > stevecassidy > nltktools

annotate g_stemmer.py @ 2:a47980ef2b96 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
author stevecassidy
date Wed, 01 Nov 2017 01:19:55 -0400
parents fb617586f4b2
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
1 import nltk
2
a47980ef2b96 planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
stevecassidy
parents: 1
diff changeset
2 from nltk.stem import PorterStemmer, LancasterStemmer, snowball
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
3 import argparse
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
4
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
5
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
6 def arguments():
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
7 parser = argparse.ArgumentParser(description="Segments the text input into separate sentences")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
8 parser.add_argument('--input', required=True, action="store", type=str, help="input text file")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
9 parser.add_argument('--output', required=True, action="store", type=str, help="output file path")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
10 parser.add_argument('--stemmer', required=False, action="store", type=str, help="output file path")
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
11 args = parser.parse_args()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
12 return args
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
13
2
a47980ef2b96 planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
stevecassidy
parents: 1
diff changeset
14
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
15 def stem_file(in_file, out_file, stemmer_type):
1
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
16 with open(in_file, 'r') as fd:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
17 unsegmented = fd.read()
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
18
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
19 with open(out_file, 'w') as output:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
20 sentences = nltk.sent_tokenize(unsegmented)
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
21 stemmer = get_stemmer(stemmer_type)
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
22 for sentence in sentences:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
23 words = nltk.word_tokenize(sentence)
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
24 for word in words:
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
25 stemmed_word = stemmer.stem(word)
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
26 output.write(stemmed_word)
fb617586f4b2 planemo upload commit a81826fe44f09a3710a35c183aa88b745aeec064-dirty
stevecassidy
parents: 0
diff changeset
27 output.write('\n')
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
28
2
a47980ef2b96 planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
stevecassidy
parents: 1
diff changeset
29
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
30 def get_stemmer(stemmer_type):
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
31 if stemmer_type == 'lancaster':
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
32 stemmer = LancasterStemmer()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
33 elif stemmer_type == 'porter':
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
34 stemmer = PorterStemmer()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
35 else:
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
36 stemmer = snowball.EnglishStemmer()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
37 return stemmer
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
38
2
a47980ef2b96 planemo upload for repository https://github.com/Alveo/alveo-galaxy-tools commit b5b26e9118f2ad8af109d606746b39a5588f0511-dirty
stevecassidy
parents: 1
diff changeset
39
0
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
40 if __name__ == '__main__':
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
41 args = arguments()
e991d4e60c17 planemo upload commit 0203cb3a0b40d9348674b2b098af805e2986abca-dirty
stevecassidy
parents:
diff changeset
42 stem_file(args.input, args.output, args.stemmer)