Copyright	(c) CNRS 2017 - present
License	AGPL + CECILL v3
Maintainer	team@gargantext.org
Stability	experimental
Portability	POSIX
Safe Haskell	Safe-Inferred
Language	Haskell2010

Gargantext.Core.Text.Terms

Description

An n-gram is a contiguous sequence of n items from a given sample of text. In Gargantext application the items are words, n is a non negative integer.

Using Latin numerical prefixes, an n-gram of size 1 is referred to as a "unigram"; size 2 is a "bigram" (or, less commonly, a "digram"); size 3 is a "trigram". English cardinal numbers are sometimes used, e.g., "four-gram", "five-gram", and so on.

Source: https://en.wikipedia.org/wiki/Ngrams

TODO group Ngrams -> Tree compute occ by node of Tree group occs according groups

compute cooccurrences compute graph

Synopsis

data TermType lang
- = Mono {
  - _tt_lang :: !lang
  }
- | Multi {
  - _tt_lang :: !lang
  }
- | MonoMulti {
  - _tt_lang :: !lang
  }
- | Unsupervised {
  - _tt_lang :: !lang
  - _tt_windowSize :: !Int
  - _tt_ngramsSize :: !Int
  - _tt_model :: !(Maybe (Tries Token ()))
  }
tt_windowSize :: forall lang. Traversal' (TermType lang) Int
tt_ngramsSize :: forall lang. Traversal' (TermType lang) Int
tt_model :: forall lang. Traversal' (TermType lang) (Maybe (Tries Token ()))
tt_lang :: forall lang lang. Lens (TermType lang) (TermType lang) lang lang
extractTerms :: NLPServerConfig -> TermType Lang -> [Text] -> IO [[TermsWithCount]]
withLang :: (Foldable t, Functor t, HasText h) => TermType Lang -> t h -> TermType Lang
data ExtractedNgrams
- = SimpleNgrams {
  - unSimpleNgrams :: Ngrams
  }
- | EnrichedNgrams {
  - unEnrichedNgrams :: NgramsPostag
  }
class ExtractNgramsT h where
- extractNgramsT :: HasText h => NLPServerConfig -> TermType Lang -> h -> DBCmd err (HashMap ExtractedNgrams (Map NgramsType Int, TermsCount))
enrichedTerms :: Lang -> PosTagAlgo -> POS -> Terms -> NgramsPostag
cleanNgrams :: Int -> Ngrams -> Ngrams
cleanExtractedNgrams :: Int -> ExtractedNgrams -> ExtractedNgrams
extracted2ngrams :: ExtractedNgrams -> Ngrams
insertExtractedNgrams :: [ExtractedNgrams] -> DBCmd err (HashMap Text NgramsId)
isSimpleNgrams :: ExtractedNgrams -> Bool
terms :: NLPServerConfig -> TermType Lang -> Text -> IO [TermsWithCount]
type WindowSize = Int
type MinNgramSize = Int
termsUnsupervised :: TermType Lang -> Text -> [TermsWithCount]
newTries :: Int -> Text -> Tries Token ()
uniText :: Text -> [[Text]]
text2term :: Lang -> [Text] -> Terms
isPunctuation :: Text -> Bool

Documentation

data TermType lang #

Constructors

Mono
Fields _tt_lang :: !lang
Multi
Fields _tt_lang :: !lang
MonoMulti
Fields _tt_lang :: !lang
Unsupervised
Fields _tt_lang :: !lang _tt_windowSize :: !Int _tt_ngramsSize :: !Int _tt_model :: !(Maybe (Tries Token ()))

Instances

Instances details

Generic (TermType lang) #
Instance details Defined in Gargantext.Core.Text.Terms Associated Types type Rep (TermType lang) :: Type -> Type # Methods from :: TermType lang -> Rep (TermType lang) x # to :: Rep (TermType lang) x -> TermType lang #
Show lang => Show (TermType lang) #
Instance details Defined in Gargantext.Core.Text.Terms Methods showsPrec :: Int -> TermType lang -> ShowS # show :: TermType lang -> String # showList :: [TermType lang] -> ShowS #
type Rep (TermType lang) #
Instance details Defined in Gargantext.Core.Text.Terms type Rep (TermType lang) = D1 ('MetaData "TermType" "Gargantext.Core.Text.Terms" "gargantext-0.0.7.1.5.3-inplace" 'False) ((C1 ('MetaCons "Mono" 'PrefixI 'True) (S1 ('MetaSel ('Just "_tt_lang") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 lang)) :+: C1 ('MetaCons "Multi" 'PrefixI 'True) (S1 ('MetaSel ('Just "_tt_lang") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 lang))) :+: (C1 ('MetaCons "MonoMulti" 'PrefixI 'True) (S1 ('MetaSel ('Just "_tt_lang") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 lang)) :+: C1 ('MetaCons "Unsupervised" 'PrefixI 'True) ((S1 ('MetaSel ('Just "_tt_lang") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 lang) :: S1 ('MetaSel ('Just "_tt_windowSize") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int)) :: (S1 ('MetaSel ('Just "_tt_ngramsSize") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 Int) :*: S1 ('MetaSel ('Just "_tt_model") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 (Maybe (Tries Token ())))))))

tt_windowSize :: forall lang. Traversal' (TermType lang) Int #

tt_ngramsSize :: forall lang. Traversal' (TermType lang) Int #

tt_model :: forall lang. Traversal' (TermType lang) (Maybe (Tries Token ())) #

tt_lang :: forall lang lang. Lens (TermType lang) (TermType lang) lang lang #

extractTerms :: NLPServerConfig -> TermType Lang -> [Text] -> IO [[TermsWithCount]] #

Sugar to extract terms from text (hidding mapM from end user). extractTerms :: Traversable t => TermType Lang -> t Text -> IO (t [Terms])

withLang :: (Foldable t, Functor t, HasText h) => TermType Lang -> t h -> TermType Lang #

data ExtractedNgrams #

Constructors

SimpleNgrams
Fields unSimpleNgrams :: Ngrams
EnrichedNgrams
Fields unEnrichedNgrams :: NgramsPostag

Instances

Instances details

Generic ExtractedNgrams #
Instance details Defined in Gargantext.Core.Text.Terms Associated Types type Rep ExtractedNgrams :: Type -> Type # Methods from :: ExtractedNgrams -> Rep ExtractedNgrams x # to :: Rep ExtractedNgrams x -> ExtractedNgrams #
Show ExtractedNgrams #
Instance details Defined in Gargantext.Core.Text.Terms Methods showsPrec :: Int -> ExtractedNgrams -> ShowS # show :: ExtractedNgrams -> String # showList :: [ExtractedNgrams] -> ShowS #
Eq ExtractedNgrams #
Instance details Defined in Gargantext.Core.Text.Terms Methods (==) :: ExtractedNgrams -> ExtractedNgrams -> Bool # (/=) :: ExtractedNgrams -> ExtractedNgrams -> Bool #
Ord ExtractedNgrams #
Instance details Defined in Gargantext.Core.Text.Terms Methods compare :: ExtractedNgrams -> ExtractedNgrams -> Ordering # (<) :: ExtractedNgrams -> ExtractedNgrams -> Bool # (<=) :: ExtractedNgrams -> ExtractedNgrams -> Bool # (>) :: ExtractedNgrams -> ExtractedNgrams -> Bool # (>=) :: ExtractedNgrams -> ExtractedNgrams -> Bool # max :: ExtractedNgrams -> ExtractedNgrams -> ExtractedNgrams # min :: ExtractedNgrams -> ExtractedNgrams -> ExtractedNgrams #
Hashable ExtractedNgrams #
Instance details Defined in Gargantext.Core.Text.Terms Methods hashWithSalt :: Int -> ExtractedNgrams -> Int # hash :: ExtractedNgrams -> Int #
type Rep ExtractedNgrams #
Instance details Defined in Gargantext.Core.Text.Terms type Rep ExtractedNgrams = D1 ('MetaData "ExtractedNgrams" "Gargantext.Core.Text.Terms" "gargantext-0.0.7.1.5.3-inplace" 'False) (C1 ('MetaCons "SimpleNgrams" 'PrefixI 'True) (S1 ('MetaSel ('Just "unSimpleNgrams") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedStrict) (Rec0 Ngrams)) :+: C1 ('MetaCons "EnrichedNgrams" 'PrefixI 'True) (S1 ('MetaSel ('Just "unEnrichedNgrams") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedStrict) (Rec0 NgramsPostag)))

class ExtractNgramsT h where #

A typeclass that represents extracting ngrams from an entity.

Methods

extractNgramsT :: HasText h => NLPServerConfig -> TermType Lang -> h -> DBCmd err (HashMap ExtractedNgrams (Map NgramsType Int, TermsCount)) #

Instances

Instances details

ExtractNgramsT HyperdataContact #
Instance details Defined in Gargantext.Database.Action.Flow.Extract Methods extractNgramsT :: HasText HyperdataContact => NLPServerConfig -> TermType Lang -> HyperdataContact -> DBCmd err (HashMap ExtractedNgrams (Map NgramsType Int, TermsCount)) #
ExtractNgramsT HyperdataDocument #	Main ngrams extraction functionality. For NgramsTerms, this calls NLP server under the hood. For Sources, Institutes, Authors, this uses simple split on " ".
Instance details Defined in Gargantext.Database.Action.Flow.Extract Methods extractNgramsT :: HasText HyperdataDocument => NLPServerConfig -> TermType Lang -> HyperdataDocument -> DBCmd err (HashMap ExtractedNgrams (Map NgramsType Int, TermsCount)) #
(ExtractNgramsT a, HasText a) => ExtractNgramsT (Node a) #
Instance details Defined in Gargantext.Database.Action.Flow.Extract Methods extractNgramsT :: HasText (Node a) => NLPServerConfig -> TermType Lang -> Node a -> DBCmd err (HashMap ExtractedNgrams (Map NgramsType Int, TermsCount)) #

enrichedTerms :: Lang -> PosTagAlgo -> POS -> Terms -> NgramsPostag #

cleanNgrams :: Int -> Ngrams -> Ngrams #

cleanExtractedNgrams :: Int -> ExtractedNgrams -> ExtractedNgrams #

extracted2ngrams :: ExtractedNgrams -> Ngrams #

insertExtractedNgrams :: [ExtractedNgrams] -> DBCmd err (HashMap Text NgramsId) #

isSimpleNgrams :: ExtractedNgrams -> Bool #

terms :: NLPServerConfig -> TermType Lang -> Text -> IO [TermsWithCount] #

Terms from Text Mono : mono terms Multi : multi terms MonoMulti : mono and multi TODO : multi terms should exclude mono (intersection is not empty yet)

type WindowSize = Int #

type MinNgramSize = Int #

termsUnsupervised :: TermType Lang -> Text -> [TermsWithCount] #

Unsupervised ngrams extraction language agnostic extraction TODO: newtype BlockText

newTries :: Int -> Text -> Tries Token () #

uniText :: Text -> [[Text]] #

TODO removing long terms > 24

text2term :: Lang -> [Text] -> Terms #

isPunctuation :: Text -> Bool #