Basically this code is for an upvote/downvote system and I'm basically

• Incrementing the count by 1 when voting up
• Decrementing the count by 1 when voting down
• If the number of downvotes > upvotes, we'll assume it's a negative score, so the count stays 0
• Reverting the count back to what it originally was when clicking upvote twice or downvote twice
• Never go below 0 (by showing negative numbers);

Basically it's the same scoring scheme reddit uses, and I tried to get some ideas from the source which was minified and kind of hard to grok:

    a.fn.vote = function(b, c, e, j) {
    if (reddit.logged && a(this).hasClass("arrow")) {
        var k = a(this).hasClass("up") ? 1 : a(this).hasClass("down") ? -1 : 0, v = a(this).all_things_by_id(), p = v.children().not(".child").find(".arrow"), q = k == 1 ? "up" : "upmod";
        p.filter("." + q).removeClass(q).addClass(k == 1 ? "upmod" : "up");
        q = k == -1 ? "down" : "downmod";
        p.filter("." + q).removeClass(q).addClass(k == -1 ? "downmod" : "down");
        reddit.logged && (v.each(function() {
            var b = a(this).find(".entry:first, .midcol:first");
            k > 0 ? b.addClass("likes").removeClass("dislikes unvoted") : k < 0 ? b.addClass("dislikes").removeClass("likes unvoted") : b.addClass("unvoted").removeClass("likes dislikes")
        }), a.defined(j) || (j = v.filter(":first").thing_id(), b += e ? "" : "-" + j, a.request("vote", {id: j,dir: k,vh: b})));
        c && c(v, k)
    }
};

I'm trying to look for a pattern, but there are a bunch of edge cases that I've been adding in, and it's still a little off.

My code (and fiddle):

$(function() {
        var down = $('.vote-down');
        var up = $('.vote-up');
        var direction = up.add(down);
        var largeCount = $('#js-large-count');
        var totalUp = $('#js-total-up');
        var totalDown = $('#js-total-down');
        var totalUpCount = parseInt(totalUp.text(), 10); 
        var totalDownCount = parseInt(totalDown.text(), 10); 

        var castVote = function(submissionId, voteType) {
            /*
            var postURL = '/vote';

            $.post(postURL, {
                submissionId : submissionId,
                voteType : voteType
            } , function (data){
                if (data.response === 'success') {
                    totalDown.text(data.downvotes);
                    totalUp.text(data.upvotes);
                }    
            }, 'json');
            */
            alert('voted!');
        };   

        $(direction).on('click', direction, function () { 
            // The submission ID
            var $that = $(this),
                submissionId = $that.attr('id'),
                voteType = $that.attr('dir'), // what direction was voted? [up or down]
                isDown = $that.hasClass('down'),
                isUp = $that.hasClass('up'),
                curVotes = parseInt($that.parent().find('div.count').text(), 10); // current vote

            castVote(submissionId, voteType);

            // Voted up on submission
            if (voteType === 'up') {
                var alreadyVotedUp = $that.hasClass('likes'),
                    upCount = $that.next('div.count'),
                    dislikes = $that.nextAll('a').first(); // next anchor attr

                if (alreadyVotedUp) {
                    // Clicked the up arrow and previously voted up
                    $that.toggleClass('likes up');
                    if (totalUpCount > totalDownCount) {
                        upCount.text(curVotes - 1);
                        largeCount.text(curVotes - 1);
                    } else {
                        upCount.text(0);
                        largeCount.text(0);
                    }    
                    upCount.css('color', '#555').hide().fadeIn();
                    largeCount.hide().fadeIn();
                } else if (dislikes.hasClass('dislikes')) {
                    // Voted down now are voting up
                    if (totalDownCount > totalUpCount) {
                        upCount.text(0);
                        largeCount.text(0);
                    } else if (totalUpCount > totalDownCount) {
                    console.log(totalDownCount);
                    console.log(totalUpCount);
                        if (totalDownCount === 0) { 
                            upCount.text(curVotes + 1);
                            largeCount.text(curVotes + 1);
                        } else {
                        upCount.text(curVotes + 2);
                        largeCount.text(curVotes + 2);
                        }    
                    } else {
                        upCount.text(curVotes + 1);
                        largeCount.text(curVotes + 1);
                    }    
                    dislikes.toggleClass('down dislikes');
                    upCount.css('color', '#296394').hide().fadeIn(200);
                    largeCount.hide().fadeIn();
                } else {
                    if (totalDownCount > totalUpCount) {
                        upCount.text(0);
                        largeCount.text(0);
                    } else {
                        // They clicked the up arrow and haven't voted up yet
                        upCount.text(curVotes + 1);
                        largeCount.text(curVotes + 1).hide().fadeIn(200);
                        upCount.css('color', '#296394').hide().fadeIn(200);
                    }    
                }    
                // Change arrow to dark blue
                if (isUp) {
                    $that.toggleClass('up likes');
                }    
            }
                       // Voted down on submission
            if (voteType === 'down') {
                var alreadyVotedDown = $that.hasClass('dislikes'),
                    downCount = $that.prev('div.count');
                // Get previous anchor attribute
                var likes = $that.prevAll('a').first();

                if (alreadyVotedDown) {
                    if (curVotes === 0) {
                        if (totalDownCount > totalUp) {
                            downCount.text(curVotes);
                            largeCount.text(curVotes);
                        } else {
                            if (totalUpCount < totalDownCount || totalUpCount == totalDownCount) {
                                downCount.text(0);
                                largeCount.text(0);
                            } else {
                                downCount.text((totalUpCount - totalUpCount) + 1);
                                largeCount.text((totalUpCount - totalUpCount) + 1);
                            }
                        }
                    } else {
                        downCount.text(curVotes + 1);
                        largeCount.text(curVotes + 1);
                    }
                    $that.toggleClass('down dislikes');
                    downCount.css('color', '#555').hide().fadeIn(200);
                    largeCount.hide().fadeIn();
                } else if (likes.hasClass('likes')) {
                    // They voted up from 0, and now are voting down
                    if (curVotes <= 1) {
                        downCount.text(0);
                        largeCount.text(0);
                    } else {
                        // They voted up, now they are voting down (from a number > 0)
                        downCount.text(curVotes - 2);
                        largeCount.text(curVotes - 2);
                    }
                    likes.toggleClass('up likes');
                    downCount.css('color', '#ba2a2a').hide().fadeIn(200);
                    largeCount.hide().fadeIn(200);
                } else {
                    if (curVotes > 0) {
                        downCount.text(curVotes - 1);
                        largeCount.text(curVotes - 1);
                    } else {
                        downCount.text(curVotes);
                        largeCount.text(curVotes);
                    }
                    downCount.css('color', '#ba2a2a').hide().fadeIn(200);
                    largeCount.hide().fadeIn(200);
                }
                // Change the arrow to red
                if (isDown) {
                    $that.toggleClass('down dislikes');
                }
            }
            return false;
        });
});?

Pretty convoluted, right? Is there a way to do something similar but in about 1/3 of the code I've written? After attempting to re-write it, I find myself doing the same thing so I just gave up halfway through and decided to ask for some help (fiddle of most recent).