.........

tf idf t

MATLAB \index{}

\index{}

tf.idf MATLAB

N grams

https://www.ncbi.nlm.nih.gov/pubmed/

http://www.brainmap.org/pubs/

https://www.ebay.com/

https://www.nlm.nih.gov/bsd/pmresources.html

/ http://www.lextek.com/manuals/onix/stopwords2.html https://www.ling.upenn.edu/courses/fall_2003/ling001/penn_treebank_pos.html

http://people.scs.carleton.ca/~armyunis/projects/kapi/porter.pdf

N N N ń ż k k + 1 k + 1 k + 1

tf idf http://people.csail.mit.edu/torralba/shortcourserloc/

j n d j = (t 1,j, t 2,j,..., t n,j ) n d j t i https://www.slideshare.net/minhahwang/introduction-to-text-mining-32058520

1 2 D 1 D 2 D 3 D 4 d t tf(d, t) t d

df(t) t w w(d, t) = tf(d, t) d tf = (tf 1, tf 2, tf 3,..., tf n ) tf(d, t) = n(d, t) ntf(d, t) ntf(d, t) = n(d, t) max t n(d, t) max t n(d, t) d

(idf) idf(t) = log D df(t) idf(t) = log 1 + D df(t) idf nidf nidf = idf logd t d tf(t) idf(t) w(d, t) = tf(d, t) idf(t) w(d, t) = ntf(d, t) nidf(t) = n(d, t) max t n(d, t) idf logd tf idf D 1 D 2 D 3 Q D IDF = log(d/df i )

tf idf m n w i,j t i d j m»n w 1,1 w 1,2 w 1,3... w 1,n w 2,1 w 2,2 w 2,3... w 2,n A = w m,1 w m,2 w m,3... w m,n m

f(x, y) = f(y, x) f(x, y) f(x, z) + f(z, y) f(x, y) 0 x y f(x, x) = 0 m

d 1 d 2 q sim(d, q) = cos( d, q) = m d i q i i=1 m m d 2 i i=1 qi 2 i=1 θ 1 θ 2 d 1 d 2 d 1 d 2 L 1 L 2 ( m )1/p dist p (d, q) = d i q i p i=1

N N N T w 1 w 2 w 3 w 4 w 5 w i w 1 w 2 w 3 w 4 w 5 w 1 w 2 w 2 w 3 w 3 w 4 w 4 w 5

w 1 w 2 w 3 w 2 w 3 w 4 w 3 w 4 w 5 N P (w1 n ) w1 n = w 1, w 2,..., w n (w n 1 ) = P (w 1 )P (w 2 w 1 )P (w 3 w 2 1)P (w 4 w 3 1)...P (w n w n 1 1 ) = n k=1 P (w k w k 1 1 ) P (w n w1 n 1 ) n (w n w n 1 1 ) P (w n w n 1, w n 1 ) (w n w n 1 1 ) P (w n w n 1 n N 1)

n (w1 n ) P (w k w k 1 ) k=1 (w n w n 1 ) = count(w n 1w n ) count(wn 1 w) (w n w n 1 ) = count(w n 1w n ) count(w n 1 ) (w 3 w 1, w 2 ) = count(w 1, w 2, w 3 ) count(w 1, w 2 ) n N+1w n ) n N+1) (w n w n +1 ) = count(wn 1 count(w n 1 N N

N S #N Grams S = W (N 1) W S

C c i,j j C

C = c 1,1 c 1,2 c 1,3... c 1,ndocs c 2,1 c 2,2 c 2,3... c 2,ndocs c N Grams,1 c N Grams,2 c N Grams,3... c N Grams,ndocs function [ngrams, C, prob_global] = test_bigrams(words,... sentence_docs, N, n_docs, stop, min_length,max_length, theme) for i = 1:length(sentences) if length(sentence(i)) > N+1) words = textscan(sentence(i), '%s') for j = 1:length(words) if ismember(words(j), stop) ismember(words(j+1),... stop) continue; else create all_ngram, docs; end end end end [un_ngrams, docs] = unique_ngrams(all_ngrams, docs); unique_words = unique(all_words); for i = 1:size(un_ngrams), t = un_ngrams{i, 1}; % ckeck token's length if length(t)<min_length length(t)>max_length, removed_words(i)=1; continue; end end doc = doc(removed_words == 0); un_ngrams = un_ngrams(removed_words == 0); %remove alphanumeric

for i = 1:size(un_ngrams), exp = '[!?!#@$%^&*_+"()[]{}:.;< -\]'; n_parts=regexp(un_ngrams,'\d+'); % identify any numeric part m_parts= regexp(un_ngrams{i,1},exp); if isempty(n_parts) == 0, % if there is a numeric part rem_terms_alphanumeric(i) = 1; end if isempty(m_parts) == 0 rem_terms_alphanumeric(i) = 1; end continue; end

P recision tp tp + fp Recall tp tp + fn F measure 2 P recision Recall P recision + Recall

200 0.05 = 10

http://www.backwordsindexing.com/

https://www.indexres.com/ http://www.masterindexing.com/home/macrex http://www.sky-software.com/ https://www.texyz.com/ http://www.fsatools.com/ https://github.com/longhunt/indexmeister

MATLAB http://scgroup20.ceid.upatras.gr:8000/tmg/

MATLAB t

http://www.cs.utexas.edu/users/dml/software/mc/ http://www.lemurproject.org/ https://www.cs.cmu.edu/~mccallum/bow/ https://sites.google.com/site/nmftool/ http://cogsys.imm.dtu.dk/toolbox/nmf/ https://cran.r-project.org/web/packages/nmfn/index.html http://glaros.dtc.umn.edu/gkhome/views/cluto https://cran.r-project.org/web/packages/nmf/index.html www.cs.waikato.ac.nz/ml/weka/ https://radimrehurek.com/gensim/ http://www.nltk.org/ https://stanfordnlp.github.io/corenlp/index.html https://cran.r-project.org/web/packages/tm/tm.pdf https://github.com/zelandiya/maui http://mallet.cs.umass.edu/ https://rapidminer.com/ https://www.textrazor.com/ https://www.sas.com/en_us/software/text-miner.html https://provalisresearch.com/

P (w 2 w 1 ) = count(w 1w 2 ) count(w 1 )

bigrams doc P (w 3 w 1 w 2 ) = count(w 1w 2 w 3 ) count(w 1 w 2 )

https://github.com/musically-ut/matlab-stanford-postagger MATLAB TaggedWords 12000 https://nlp.stanford.edu/software/tagger.shtml

https://github.com/aneesha/rake/

https://www.theiet.org/resources/inspec/

MATLAB IndexedBigrams IndexedTrigrams IndexedUnigrams \index{} \index{} addition deletion completely unrelated document

p 0 p 1

tagged_bigrams tagged_trigrams

recall = 18 24 = 0.75 recall = 8 24 = 0.333 https://kevinastraight.wordpress.com/indexmeister/

# # # # http://www.island.net/~hamill/tips_to_authors_and_editors.htm

# # # #

MATLAB

http://www. dlsi.ua.es/~elloret/publications/textsummarization.pdf https://www. manning.com/books/natural-language-processing-in-action

https://nlp.stanford.edu/ir-book/ https: //web.stanford.edu/~jurafsky/slp3/ed3book.pdf

http://www.minerazzi.com/tutorials/ term-vector-3.pdf https://www.codeproject.com/articles/439890/ Text-Documents-Clustering-using-K-Means-Algorithm

p unigram = 0.001 p bi gram = 0.5 p tr gram = 0.5

p unigram = 0.001 p bi gram = 0.5 p tr gram = 0.5

p unigram = 0.001 p bi gram = 0.5 p tr gram = 0.5

p unigram = 0.001 p bi gram = 0.5 p tr gram = 0.5