def my_mega_function(snippet, query)
    """This takes a snippet of text, and a query parameter and returns """

    #Logic goes here, and often runs on for several hundred lines
    #There are often deeply nested conditional statements and loops
    #Function could reach several hundred, if not thousands of lines
    
    return result

% python pygenie.py complexity --verbose highlight spy
File: /Users/ngift/Documents/src/highlight.py
Type Name                                                                   Complexity 
----------------------------------------------------------------------------------------
M    HighlightDocumentOperations._create_snippit                                  3
M    HighlightDocumentOperations._reconstruct_document_string                     3
M    HighlightDocumentOperations._doc_to_sentences                                2
M    HighlightDocumentOperations._querystring_to_dict                             2
M    HighlightDocumentOperations._word_frequency_sort                             2
M    HighlightDocumentOperations.highlight_doc                                    2
X    /Users/ngift/Documents/src/highlight.py 1          
C    HighlightDocumentOperations                                                  1
M    HighlightDocumentOperations.__init__                                         1
M    HighlightDocumentOperations._custom_highlight_tag                            1
M    HighlightDocumentOperations._score_sentences                                 1
M    HighlightDocumentOperations._multiple_string_replace                         1

% nosetests -v --with-coverage --cover-package=highlight --with-doctest\
     --cover-erase --exe

Doctest: highlight.HighlightDocumentOperations._custom_highlight_tag ... ok
test_functional.test_snippit_algorithm ... ok
test_custom_highlight_tag (test_highlight.TestHighlight) ... ok
Consumes the generator, and then verifies the result[0] ... ok
Verifies highlighted text is what we expect ... ok
test_multi_string_replace (test_highlight.TestHighlight) ... ok
Verifies the yielded results are what is expected ... ok

Name        Stmts   Exec  Cover   Missing
-----------------------------------------
highlight      71     71   100%   
----------------------------------------------------------------------
Ran 7 tests in 4.223s

OK

#/usr/bin/python
# -*- coding: utf-8 -*-

"""
:mod:`highlight` -- Highlight Methods
===================================

.. module:: highlight
   :platform: Unix, Windows
   :synopsis: highlight document snippets that match a query.
.. moduleauthor:: Noah Gift

 
Requirements::
    1.  You will need to install the ntlk library to run this code.
        http://www.nltk.org/download
    2.  You will need to download the data for the ntlk:
        See http://www.nltk.org/data::
        
        import nltk
        nltk.download()

"""

import re
import logging

import nltk

#Globals
logging.basicConfig()
LOG = logging.getLogger("highlight")
LOG.setLevel(logging.INFO)

class HighlightDocumentOperations(object):

    """Highlight Operations for a Document"""
    
    def __init__(self, document=None, query=None):
        """
        Kwargs:
            document (str):
            query (str):
            
        """
        self._document = document
        self._query = query
    
    @staticmethod
    def _custom_highlight_tag(phrase,
                              start="<strong>",
                              end="</strong>"):
        
        """Injects an open and close highlight tag after a word

        Args:
            phrase (str) - A word or phrase.
        Kwargs:
        start (str) - An opening tag.  Defaults to <strong>
        end (str) - A closing tag.  Defaults to </strong>
        Returns:
            (str) word or phrase with custom opening and closing tags
            
        >>> h = HighlightDocumentOperations()
        >>> h._custom_highlight_tag("foo")
        'foo'
        >>>
        
        """
        tagged_phrase = "{0}{1}{2}".format(start, phrase, end)
        return tagged_phrase
    
    def _doc_to_sentences(self):
        """Takes a string document and converts it into a list of sentences
        
        Unfortunately, this approach might be a tad naive for production
        because some segments that are split on a period are really an
        abbreviation, and to make things even more complicated, an
        abbreviation can also be the end of a sentence::
            http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html
        
        Returns:
            (generator) A generator object of a tokenized sentence tuple,
            with the list position of sentence as the first portion of
            the tuple, such as:  (0, "This was the first sentence")
        
        """
        
        tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
        sentences = tokenizer.tokenize(self._document)
        for sentence in enumerate(sentences):
            yield sentence

    @staticmethod    
    def _score_sentences(sentence, querydict):
        """Creates a scoring system for each sentence by substitution analysis
        
        Tokenizes each sentence, counts characters
        in sentence, and pass it back as nested tuple
    
        Returns:
            (tuple) - (score (int), (count (int), position (int),
                   raw sentence (str))
            
        """
        
        position, sentence = sentence
        count = len(sentence)
        regex = re.compile('|'.join(map(re.escape, querydict)))
        score = len(re.findall(regex, sentence))
        processed_score = (score, (count, position, sentence))
        return processed_score
    
    def _querystring_to_dict(self, split_token="+"):
        """Converts query parameters into a dictionary
        
        Returns:
            (dict)- dparams, a dictionary of query parameters
            
        """
        
        params = self._query.split(split_token)
        dparams = dict([(key, self._custom_highlight_tag(key)) for\
                    key in params])
        return dparams
    
    @staticmethod
    def _word_frequency_sort(sentences):
        """Sorts sentences by score frequency, yields sorted result
        
        This will yield the highest score count items first.
        
        Args:
            sentences (list) - a nested tuple inside of list
            [(0, (90, 3, "The crust/dough was just way too effin' dry for me.
            Yes, I know what 'cornmeal' is, thanks."))]

        """

        sentences.sort()
        while sentences:
            yield sentences.pop()

    def _create_snippit(self, sentences, max_characters=175):
        """Creates a snippet from a sentence while keeping it under max_chars 
        
        Returns a sorted list with max characters.  The sort is an attempt
        to rebuild the original document structure as close as possible,
        with the new sorting by scoring and the limitation of max_chars.
        
        Args:
            sentences (generator) - sorted object to turn into a snippit
            max_characters (int) - optional max characters of snippit
           
        Returns:
            snippit (list) - returns a sorted list with a nested tuple that
            has the first index holding the original position of the list::
            
            [(0, (90, 3, "The crust/dough was just way too effin' dry for me.
            Yes, I know what 'cornmeal' is, thanks."))]
            
        """
        
        snippit = []
        total = 0
        for sentence in self._word_frequency_sort(sentences):
            LOG.debug("Creating snippit", sentence)
            score, (count, position, raw_sentence) = sentence
            total += count
            if total < max_characters:
                #position now gets converted to index 0 for sorting later
                snippit.append(((position), score, count, raw_sentence))
        
        #try to reassemble document by original order by doing a simple sort
        snippit.sort()
        return snippit
    
    @staticmethod
    def _multiple_string_replace(string_to_replace, dict_patterns):
        """Performs a multiple replace in a string with dict pattern.
        
        Borrowed from Python Cookbook.
        
        Args:
            string_to_replace (str) - String to be multi-replaced
            dict_patterns (dict) - A dict full of patterns
            
        Returns:
            (str) - Multiple replaced string.
        
        """
        
        regex = re.compile('|'.join(map(re.escape, dict_patterns)))
        def one_xlat(match):
            """Closure that is called repeatedly during multi-substitution.
            
            Args:
                match (SRE_Match object)
            Returns:
                partial string substitution (str)
            
            """
            
            return dict_patterns[match.group(0)]
        
        return regex.sub(one_xlat, string_to_replace)
    
    def _reconstruct_document_string(self, snippit, querydict):
        """Reconstructs string snippit, build tags, and return string
        
        A helper function for highlight_doc.
        
        Args:
            string_to_replace (list) - A list of nested tuples, containing
            this pattern::
            
            [(0, (90, 3, "The crust/dough was just way too effin' dry for me.
            Yes, I know what 'cornmeal' is, thanks."))]
            
            dict_patterns (dict) - A dict full of patterns
        
        Returns:
            (str) The most relevant snippet with the query terms highlighted.
        
        """
        
        snip = []
        for entry in snippit:
            score = entry[1]
            sent = entry[3]
            #if we have matches, now do the multi-replace
            if score:
                sent = self._multiple_string_replace(sent,
                                                    querydict)
            snip.append(sent)
        highlighted_snip = " ".join(snip)
        
        return highlighted_snip
        
    def highlight_doc(self):
        """Finds the most relevant snippit with the query terms highlighted
        
        Returns:
            (str) The most relevant snippet with the query terms highlighted.
        
        """
        
        #tokenize to sentences, and convert query to a dict
        sentences = self._doc_to_sentences()
        querydict = self._querystring_to_dict()
        
        #process and score sentences
         scored_sentences = []
        for sentence in sentences:
            scored = self._score_sentences(sentence, querydict)
            scored_sentences.append(scored)
        
        #fit into max characters, and sort by original position
        snippit = self._create_snippit(scored_sentences)
        #assemble back into string
        highlighted_snip = self._reconstruct_document_string(snippit,
                                                             querydict)

        return highlighted_snip

#/usr/bin/python
# -*- coding: utf-8 -*-
"""
Tests this query searches a document, highlights a snippit and returns it
http://www.example.com/search?find_desc=deep+dish+pizza&ns=1&rpp=10&find_loc=\
                                                        San+Francisco%2C+CA

Contains both unit and functional tests.

"""


import unittest
from highlight import HighlightDocumentOperations

class TestHighlight(unittest.TestCase):
    
    def setUp(self):
        
        self.document = """
Review for their take-out only.
Tried their large Classic (sausage, mushroom, peppers and onions) deep dish;\
and their large Pesto Chicken thin crust pizzas.
Pizza = I've had better.  The crust/dough was just way too effin' dry for me.\
Yes, I know what 'cornmeal' is, thanks.  But it's way too dry.\
I'm not talking about the bottom of the pizza...I'm talking about the dough \
that's in between the sauce and bottom of the pie...it was like cardboard, sorry!
Wings = spicy and good.   Bleu cheese dressing only...hmmm, but no alternative\
of ranch dressing, at all.  Service = friendly enough at the counters.  
Decor = freakin' dark.  I'm not sure how people can see their food.  
Parking = a real pain.  Good luck.        
        
        """
        self.query = "deep+dish+pizza"
        self.hdo = HighlightDocumentOperations(self.document, self.query)
        
    def test_custom_highlight_tag(self):
        
        actual = self.hdo._custom_highlight_tag("foo",
                                            start="[BAR]",
                                            end="[ENDBAR]")
        expected = "[BAR]foo[ENDBAR]"
        self.assertEqual(actual,expected)
    
    def test_query_string_to_dict(self):
        """Verifies the yielded results are what is expected"""
        
        result = self.hdo._querystring_to_dict()
        expected = {"deep": "deep",
                    "dish": "dish",
                    "pizza":"pizza"}
        
        self.assertEqual(result,expected)
    
    def test_multi_string_replace(self):
        
        query = """pizza = I've had better"""
        expected = """pizza = I've had better"""
        query_dict = self.hdo._querystring_to_dict()
        result = self.hdo._multiple_string_replace(query, query_dict)
        self.assertEqual(expected, result)
        
    def test_doc_to_sentences(self):
        """Consumes the generator, and then verifies the result[0]"""
        
        results = []
        expected = (0,'\nReview for their take-out only.')
        
        for sentence in self.hdo._doc_to_sentences():
            results.append(sentence)
        self.assertEqual(results[0], expected)
 
    def test_highlight(self):
        """Verifies highlighted text is what we expect"""
        
        expected = """Tried their large Classic (sausage, mushroom, peppers and onions)\
deep dish;and their large Pesto Chicken thin crust \
pizzas."""
 
        actual = self.hdo.highlight_doc()
        self.assertEqual(expected, actual)
    
    def tearDown(self):
        
        del self.query
        del self.hdo
        del self.document

if __name__ == '__main__':
    unittest.main()

"""Functional Test That Performs Some Basic Sanity Checks"""

from highlight import HighlightDocumentOperations


def test_snippit_algorithm():
    document1 = """
        This place has awesome deep dish pizza.
        I have been getting delivery through Waiters on wheels for years.
        It is classic, deep dish  Chicago style pizza.
        Now I found out they also have half-baked to pick-up and cook at home.
        This is a great benefit. I am having it tonight. Yum.
        """
    document2 = """Review for their take-out only.
Tried their large Classic (sausage, mushroom, peppers and onions) deep dish;\
and their large Pesto Chicken thin crust pizzas.
Pizza = I've had better.  The crust/dough was just way too effin' dry for me.\
Yes, I know what 'cornmeal' is, thanks.  But it's way too dry.\
I'm not talking about the bottom of the pizza...I'm talking about the dough \
that's in between the sauce and bottom of the pie...it was like cardboard, sorry!
Wings = spicy and good.   Bleu cheese dressing only...hmmm, but no alternative\
of ranch dressing, at all.  Service = friendly enough at the counters.  
Decor = freakin' dark.  I'm not sure how people can see their food.  
Parking = a real pain.  Good luck."""
    
    h1 = HighlightDocumentOperations(document1, "deep+dish+pizza")
    actual = h1.highlight_doc()
    print "Raw Document1: %s" % document1
    print " Formatted Document1: %s" % actual
    assert  len(actual) < 500
    assert "<strong>" in actual

    h2 = HighlightDocumentOperations(document2, "deep+dish+pizza")
    actual = h2.highlight_doc()
    print "Raw Document2: %s" % document2
    print " Formatted Document2: %s" % actual
    assert  len(actual) < 500
    assert "<strong>" in actual

    
if __name__ == "__main__":
    test_snippit_algorithm()

% pylint highlight spy 
No config file found, using default configuration
************* Module highlight
E: 89:HighlightDocumentOperations._doc_to_sentences: Instance of 'unicode' has no 
    'tokenize' member (but some types could not be inferred)
E: 89:HighlightDocumentOperations._doc_to_sentences: Instance of 'ContextFreeGrammar' 
    has no 'tokenize' member (but some types could not be inferred)
W:108:HighlightDocumentOperations._score_sentences: Used builtin function 'map'
W:192:HighlightDocumentOperations._multiple_string_replace: Used builtin function 'map'
R: 34:HighlightDocumentOperations: Too few public methods (1/2)

Report
======
69 statements analysed.

Global evaluation
-----------------
Your code has been rated at 8.12/10 (previous run: 8.12/10)