SoylentNews

On IRC (in #Soylent), MrBluze and I have been discussing how to make a better messaging system. You would need to be able to easily send to individuals or groups, have messages either public or private, be able to verify that the message came from who it claims to be from (and someone who's a contact on your list), and all without relying on specific server software or a specific hosting company).

So below is a very basic proof of concept that does the following:
- Creates two identities (a sender and recipient).
- Gives them each a public and private key
- Creates a message that the recipient can verify is:
          - Intended for, and only readable by the recipient.
          - Created by the sender, and not just someone using their name.
- Formats the result in plain text (base64 encoded) so it's easy to transmit.

This means the message itself could be send via any text medium, public or private, as only the intended people will be able to read its contents and they'll know who sent it based on the message itself.

It's all very basic and preliminary but I had to figure out a few things about how to use openssl and hashes properly, so it may be instructive to other people interested in this kind of thing.

I'll get around to adding more comments at some point.

<?php

function encrypt_longer_string_public_key($source_string, $key, $chunk_length=100,$delimiter = "...")
{
    $source_array = str_split($source_string, $chunk_length);
    $result_string = "";

    foreach ($source_array as $source_chunk)
    {
        $encrypted_chunk = "";
        openssl_public_encrypt($source_chunk, $encrypted_chunk, $key);
        if ($result_string != "") $result_string.=$delimiter;
        $result_string.= base64_encode($encrypted_chunk);
    }
    return $result_string;
}

function encrypt_longer_string_private_key($source_string, $key, $chunk_length=100,$delimiter = "...")
{
    $source_array = str_split($source_string, $chunk_length);
    $result_string = "";
    foreach ($source_array as $source_chunk)
    {
        $encrypted_chunk = "";
        openssl_private_encrypt($source_chunk, $encrypted_chunk, $key);
        if ($result_string != "") $result_string.=$delimiter;
        $result_string.= base64_encode($encrypted_chunk);
    }
    return $result_string;
}

function decrypt_longer_string_private_key($encrypted_string, $key, $delimiter="...")
{
    $parts = explode($delimiter, $encrypted_string);
    $number_of_parts = count($parts);
    $result_string = "";
    foreach ($parts as $one_part)
    {
        $one_part = base64_decode($one_part);
        openssl_private_decrypt($one_part, $decrypted_part, $key);
        $result_string.=$decrypted_part;
    }
    return $result_string;
}

function decrypt_longer_string_public_key($encrypted_string, $key, $delimiter="...")
{
    $parts = explode($delimiter, $encrypted_string);
    $number_of_parts = count($parts);
    $result_string = "";
    foreach ($parts as $one_part)
    {
        $one_part = base64_decode($one_part);
        openssl_public_decrypt($one_part, $decrypted_part, $key);
        $result_string.=$decrypted_part;
    }
    return $result_string;
}

function verify_hashed_string($string_with_hash, $hash_algorithm = "sha256")
{
    $extracted_hash = substr($string_with_hash, -64);
    $extracted_text = substr($string_with_hash, 0, -64);
    $calculated_hash = hash($hash_algorithm, $extracted_text);
    if ($calculated_hash == $extracted_hash) return $extracted_text;
    else return false;

}

// 1) Create a new key:
putenv("OPENSSL_CONF=c:\\wamp\\www\\nospamplease\\openssl.cnf");

if (file_exists(getenv('OPENSSL_CONF'))) print "config file exists<br><Br>";

$config_args = array(
    'config' => getenv('OPENSSL_CONF'));

$sender_key = openssl_pkey_new($config_args);
$recipient_key = openssl_pkey_new($config_args);

// 2) Extract the private key string.
print "sender key is "; var_dump($sender_key); print "<br><br>";
print "recipient key is "; var_dump($recipient_key); print "<br><br>";

openssl_pkey_export($sender_key, $sender_private_key, NULL, $config_args);
openssl_pkey_export($recipient_key, $recipient_private_key, NULL, $config_args);

// 3) Get the public key.
$key_details = openssl_pkey_get_details($sender_key);
$sender_public_key = $key_details["key"];

$key_details = openssl_pkey_get_details($recipient_key);
$recipient_public_key = $key_details["key"];

// 4) Display the keys. The hash of the public key could be used as a "to" identifier.
print "Sender Private Key: $sender_private_key<br><br>
    Sender Public Key: $sender_public_key<BR><BR>";
print "<hr>";
print "Recipient: Private Key: $recipient_private_key<br><br>
    Recipient Public Key: $recipient_public_key<BR><BR>";

// Show some text before and after encryption and decryption
$original_text = "This is totally a secret message";

// Add a checksum to the end.
$inner_message_hash = hash("sha256", $original_text);
$hashed_inner_string = $original_text.$inner_message_hash;

// Encrypt the message with senders private key.
$inner_encrypted = encrypt_longer_string_private_key($hashed_inner_string, $sender_private_key);

// Add a checksum to the end of the encrypted messaged, after a base64_encode of it.
$outer_message_hash = hash("sha256", $inner_encrypted);

$hashed_outer_string = $inner_encrypted.$outer_message_hash;

$outer_encrypted = encrypt_longer_string_public_key($hashed_outer_string, $recipient_public_key);
//$outer_encrypted_as_string = base64_encode($outer_encrypted);

print "<br>Original message:<br>$original_text<br>";
print "<br>Hashed inner string:<br>$hashed_inner_string<br>";
print "<br>Inner string encrypte:<br>$inner_encrypted<br>";
print "<br>Outer string (inner string encrypted + hash of encrypted string)<br>$hashed_outer_string<br>";
print "<br>Outer string encrypted:<br>$outer_encrypted<br><br>";

// Now go the other way, decrypt it and show the decrypted message. Verify each step.

$outer_decrypted = decrypt_longer_string_private_key($outer_encrypted, $recipient_private_key);
$outer_decrypted = verify_hashed_string($outer_decrypted);

if ($outer_decrypted) print "outer decrypted is : ".$outer_decrypted."<br><br>";
else {print "couldn't verify outer decryption";return;}

$inner_decrypted = decrypt_longer_string_public_key($outer_decrypted, $sender_public_key);
$inner_decrypted = verify_hashed_string($inner_decrypted);
if ($inner_decrypted) print "Inner decrypted is: $inner_decrypted";
else {print "couldn't verify inner decrypted"; return;}

?>

How to find the most useless page of SoylentNews:

From the dropdown selection button below the story (the one offering "Threaded", "Nested", etc. for comments) select "No Comments", then click "Change".

The option delivers what it promises: No comments. Also no story. Just the settings bar.

Note that this is not a critique (it's clearly just an artefact of how Slashcode works, and not harmful in any way). I just considered it amusing.

So ... I've recently gotten back into minecraft, and figured that perhaps there are other MC players here at SN, so I wanted to know if there was enough interest to setup a MC server in general. I'd probably use CraftBukkit, and I'm open to running mods if others are interesting. Leave a message below if you'd be interested.

Mods I'd like to run:
  * Traincraft
  * Railcraft
  * Mystcraft (useful for getting new ores without having to reset maps; age creation would be restricted to admins though; mystcraft is a server hog).

Leave your thoughts below.

Since we've got a fair number of complaints about us running too many site news articles, I'm going to condemn this to my journal, then link it next time we *do* post something about the site. For a large portion of today (4/16), SoylentNews users had issues with commenting, and moderation was completely hosed. This was due to a backend change; we shifted the site behind a loadbalancer in preparation of bringing up a new frontend and give us considerably more redundancy and latitude with working with the backend.

This change had been setup on dev for the last week with us testing it to see what (if anything) broken, and it was discussed and signed off by all of the staff. Last night, I flipped the nodebalancer to connect to production instead of dev, then changed the DNS A record for the site to point at the loadbalancer.

I stayed up for several hours at this point to ensure nothing odd was going on, and satisfied that the world would keep spinning, I went to bed. What I found though was I broke the formkeys system. Slash knows about the X-Forwarded-By header, a mechanism for when a site is behind a proxy on how to relay client IP information (this mechanism was already used by both varnish and nginx), however, for security reasons, we strip out the XFF header from inbound connections unless its on a specific whitelist. On both dev and production, we had whitelisted the nodebalancer to pass this header in properly.

Or so we thought. Linode's documentation doesn't mention, but the IP address listed in the admin interface is *not* the IP used to connect to the site; instead it uses a special internal IP address which isn't listed or documented anywhere. Our security precautions stripped out the X-Forwarded-By header, and made it appear that all inbound users were coming from the same IP. This wasn't noticed on dev as slash ignores the formkeys system for admins, and the few of us beating on it with non-admin accounts weren't able to do enough abuse to trigger the formkey limiters.

Our peak hours are generally evenings EDT, which means the low traffic at night wasn't enough to trip it either (or at least no one on IRC poked me about it, nor were there any bugs on it on our github page. However, once traffic started picking up, users began to clobber each other, commenting broke, and the site went to straight to hell. When I got up, debugging efforts were underway, but it took considerable time to understand the cause of the breakage; simply reverting LBing wasn't an easy fix since we'd still have to wait for DNS to propagate and we needed the load balancer anyway. After a eureka moment, we were able to locate the correct internal IPs, and whitelist them, which got the site partially functional again. (we have informed Linode about this, and they said our comments are on its way to the appropriate teams; hopefully no other site will ever have this same problem).

The last remaining item was SSL; we had originally opted out of terminating SSL on the loadbalancer, prefering to do it on the nginx instance, so Port 443 was set to TCP loadbalancing. This had the same effect as there is no way for us to see the inbound IP (I had assumed it would do something like NAT to make connections appear like they were coming from the same place). The fix was utlimately installing the SSL certificate on the load balancer, then modifying varnish to look for the X-Forwarded-Proto header to know if a connection was SSL or not. I'm not hugely happy about this as it means wiretapping would be possible between the load balancer and the node, but until we have a better system for handling SSL, there isn't a lot we can do about it.

As always, leave comments below, and I'll leave my two cents.

Hi,

So, my evening/night started like any other until I started having several nosebleeds which all occurred within one or two hours or so not to mention the headache, so like any responsible geek I decided to phone my Doctor, fortunately, he was still awake and told me to go to the hospital... (what I got told there isn't important to this journal entry)

Anyway, so when I came home the IRC server was *seemingly* down, in fact, my entire server was *seemingly* down. After contacting my ISP, I had to go through a few hoops - and as it turns out, it was a faulty switch (you'd think, they would at least be monitoring their own switches right?). They replaced it and it's all back up. Incidentally, I was going to configure Carbon (which was going to be another IRC server) last night but the hospital thing came up.

So all in all, my night sucked big time.

Hopefully, not to many of you were inconvenienced by the downtime, perhaps some of you even managed to see the sun, or at the very least, felt the wind.

Update: webchat was still down, sorry about that.

Now that I've had some time to clear my head, I want to expand on my original feelings. I'm pissed off about this, and my temper flared through on the original post. I'm leaving it as is because I'm not going to edit it to make myself look better, and because it sums up my feelings pretty succinctly. How would you feel if something you worked on under the promise of building the best site for a community was regularly and routinely causing corporate firewalls and IDS systems to go off like crazy?

You'd be pissed. Had we known about this behaviour in advance, it would have been disabled at golive or in a point release, and a minor note would have gone up about it. Instead, I found out because we were tripping a user's firewall causing the site to get autoblocked. I realize some people feel this is acceptable behaviour, but a website should *never* trigger IDS or appear malicious in any way. Given the current state of NSA/GCHQ wiretapping and such, it means that anything tripping these types of systems is going to be looked at suspiciously to say the least. I'm not inherently against such a feature (IRC networks check for proxying for instance), but its clearly detailed in the MOTD of basically every network that does it.

There wasn't a single thing in the FAQ that suggested it, and a Google search against the other site didn't pop something up that dedicated what was being done; just a small note that some proxies were being blocked. Had the stock FAQ file, or documentation, or anything detailed this behaviour, while I might still have thought it wrong, at least I wouldn't have gotten upset about it. I knew that there was proxy scanning code in slashcode, but all the vars in the database were set to off; as I discovered, they're ignored leading me to write a master off switch in the underlying scanning function.

Perhaps in total, this isn't a big deal, but it felt like a slap in the face. I know I have a temper, and I've been working to keep it under wraps (something easier said than done, but nothing worthwhile is ever easy). CmdrTaco himself commented on this on hackernews and I've written a reply to him about it. Slashdot did what they felt was necessary to stop spam on their site, and by 2008, slashcode only really existed for slashdot itself; other slash sites run on their own branches of older code. Right or wrong, such behaviour should be clearly documented, as its not something you expect, and can (and has) caused issues to users and concerns due to lack of communication. Transparency isn't easy, but I have found its the only way to have a truly healthy community. Perhaps you disagree. I'll respond to any comments or criticisms left below.

Just before, while moderating, I got this enormous error message:

Internal SAN check blew up checking moderation rights. Please let the powers that be know how and where you recieved this error Internal SAN check blew up checking moderation rights. Please let the powers that be know how and where you recieved this error Internal SAN check blew up checking moderation rights. Please let the powers that be know how and where you recieved this error Internal SAN check blew up checking moderation rights. Please let the powers that be know how and where you recieved this error Internal SAN check blew up checking moderation rights. Please let the powers that be know how and where you recieved this error Internal SAN check blew up checking moderation rights. Please let the powers that be know how and where you recieved this error Internal SAN check blew up checking moderation rights. Please let the powers that be know how and where you recieved this error Internal SAN check blew up checking moderation rights. Please let the powers that be know how and where you recieved this error

I have no idea where to properly report this, though. Thus I just put it here and hope that either the right person finds it, or someone who sees it can tell me where to report.

FWIW, apart from that message, everything seemed to be fine (including moderation, reduction of mod points, and no longer being able to moderate after using up my mod points).

https://github.com/crutchy-/test/blob/master/karma_published.php

bacon+
(only single +/- to differentiate from bender)

~karma bacon

~rainbow pretty text

etc

todo: quotes

Here's a third version of my voting program. Unlike the previous version, it doesn't allow duplicate ranks (you can't rank two candidates as equal first, or equal second, etc).

It can be run on a set of text-based votes, e.g. emails or forum-comments. It recognises and counts valid votes, and filters out invalid data.

It gives a score to each candidate, based on the rank you give it. It is effectively a "Borda Count" voting system (http://en.wikipedia.org/wiki/Borda_count). Version 2 used "Range Voting" and version 1 used "Approval Voting".

I've also tidied the code up a bit, to make it easier to read and simpler to use.

The code below contains example data so you can run it as is, to get an idea of how it works.

<?php

/* How to use this script:

1 - Ask people to write votes that look like this:
    some candidate name = 1
    some other = 2
    another_one = 4
    Any other text will just be ignored.
    Any invalid = votes will be ignored, too
2 - Collect votes. e.g. via email, forum-comments, or a special web-form.
3 - Put all the votes in an array. Each vote should contain its vote-text, and a User ID.
4 - Run this script to filter, parse and count the votes.

*/

// Example candidates:
$valid_candidates = array(
    "candidateone",
    "another candidate",
    "yet another option",
    "somethingelse");

// Example votes:
$votes_array = array(
    // Upper case or lower case doesn't matter.
    array(
        "user_id"=>234,
        "text"=>"
            candidateOne = 1
            Another Candidate = 2
            SomethingElse = 3
        "),
    // Duplicate user. This will be handled properly.
    array(
        "user_id"=>234,
        "text"=>"
            Oops forgot one I like:
            Yet Another Option = 4
            Did I mention:
            CandidateOne = 1
        "
        ),
    // This one contains mostly invalid rankings, and one valid one.
    array(
        "user_id"=>345,
        "text"=>"
            // I hate CandidateOne
            CandidateOne = 6
            Another Candidate = 1
            Yet Another Option = 1
            My friend who's not listed = 3
        ")
    );

// Some options on how the votes are counted.
$allow_duplicate_ranks = false;
$allow_write_in_candidates = false;

// If you allow write-in candidates, specify a maximum possible rank.
// Otherwise the maximum rank equals the number of candidates.
if ($allow_write_in_candidates) $maximum_rank = 10;
else $maximum_rank = count($valid_candidates);

// put valid-user filter in here if necessary
function valid_user($user_id){return true;}

// Arrays to store the counted votes in:
$votes_by_voter = array();
$votes_by_candidate = array();

// Process all votes
foreach ($votes_array as $vote)
{

  // Is it a valid registered user?
  if (valid_user($vote["user_id"]))
  {

    // Process each line of the vote
    $vote_lines = explode("\n", trim($vote["text"]));
    foreach ($vote_lines as $this_line)
    {

      // Does it have an equals sign
      $equals_sign = strpos($this_line, "=");
      if ($equals_sign !== false)
      {

        // Does it have only one equals sign?
        $cleaned_up_line_text = trim($this_line, ";.!\t\n\r\0");
        $parts_of_line = explode("=", $cleaned_up_line_text);
        if (count($parts_of_line) == 2)
        {

          // Get the candidate and rank, make sure they're valid.
          $candidate = strtolower(trim($parts_of_line[0]));
          $candidate_is_valid = in_array($candidate, $valid_candidates);
          $rank = intval(trim($parts_of_line[1]));
          $rank_is_valid = ( ($rank > 0) && ($rank <= $maximum_rank) );

          // Proceed if (it's a valid rank number) and
          // (the candidate is valid, or we're allowing write-in candidates).
          if (($rank_is_valid) && ($candidate_is_valid || $allow_write_in_candidates))
          {
            // Get the score for this candidate.
            // The score is: maximum_rank - (this_rank - 1).
            // For example:
            // - Say there are 5 candidates and the maximum rank is 5
            // - A rank of 1 give it a score of 5.
            // - A rank of 2 gives it a score of 4.
            // - A rank of 5 gives a score of 1.
            // See "Borda Count".
            $score = $maximum_rank - ($rank-1);

            // If this is the voter's first vote, create a voting-record for them.
            // This keeps track of which candidates and ranks they've already voted.
            $voter = $vote["user_id"];
            if (!isset($votes_by_voter[$voter]))
            {
              $votes_by_voter[$voter]["candidates"] = array();
              $votes_by_voter[$voter]["ranks"] = array();
            }

            // Make sure this user hasn't already voted on this candidate
            if (!isset($votes_by_voter[$voter]["candidates"][$candidate]))
            {

              // Make sure the user hasn't already assigned this rank number,
              // or that we're allowing duplicate ranks.
              if ($allow_duplicate_ranks || (!isset($votes_by_voter[$voter]["ranks"][$rank])))
              {

                // Remember that this voter has voted for this candidate,
                // and has used up this rank.
                $votes_by_voter[$voter]["candidates"][$candidate] = true;
                $votes_by_voter[$voter]["ranks"][$rank] = true;

                // Count the vote towards the total for this candidate.
                if (!isset($votes_by_candidate[$candidate]))
                  $votes_by_candidate[$candidate] = $score;
                else $votes_by_candidate[$candidate] += $score;

              } // End of checking if this rank is a duplicate for this voter.
            } // End of check checking if candidate is a duplicate for this voter.
          } // End of check for valid vote values.
        } // End of check for correctly formatted vote
      } // End of check for equals sign
    } // End of for loop for lines of vote text.
  } // of check for valid user.
} // end of for loop for all votes.

print "Who have voters voted for, and which ranks have they used?:<pre>";
print_r($votes_by_voter);
print "</pre><Br>";
print "What score does each candidate end up with<pre>";
// Sort the candidates from highest to lowest
arsort($votes_by_candidate);
print_r($votes_by_candidate);
print "</pre>";

?>