I wanted to make a script to check if flatpak update app permissions have changed before updating them. I decided to use ChatGPTs newest model, 01-preview for this task and holy crap am I impressed. I had to give it 14 commands to get to the final product and it seems to work great. That number could have been reduced quite a bit had I done things a bit differently.

I still had to find the problems, look up reference docs to correct it, and even debug in an IDE a little. But just telling it where the problems were, it got there in the end, and its user interaction output is way better than what I was planning on doing.

Anywho, here are the commands I gave it, followed by the code it gave me in the end. I have not throughly checked and tested the code, but it seems to work from a few quick trials. For demonstration purposes, I have left all of my original typos uncorrected below.

<?php
/**
 * This script checks for Flatpak updates and reports permission changes.
 */

// Function to parse permissions from Flatpak metadata
function parsePermissions($content) {
    $permissions = [];
    $lines = is_array($content) ? $content : explode("\n", $content);
    $currentSection = '';
    $skipSections = ['Application']; // Sections to skip

    foreach ($lines as $line) {
        $line = trim($line);

        if (empty($line)) {
            continue;
        }

        // Check for section headers
        if (preg_match('/^\[(.*)\]$/', $line, $matches)) {
            $currentSection = $matches[1];

            // Skip the [Application] section
            if (in_array($currentSection, $skipSections)) {
                $currentSection = '';
                continue;
            }

            $permissions[$currentSection] = [];
        } elseif ($currentSection !== '') {
            // Only process lines within non-skipped sections
            $parts = explode('=', $line, 2);
            if (count($parts) == 2) {
                $key = $parts[0];
                $values = explode(';', trim($parts[1], ';'));
                $permissions[$currentSection][$key] = $values;
            } else {
                // Handle keys without '=' (e.g., single permissions)
                $permissions[$currentSection][$line] = [];
            }
        }
    }

    return $permissions;
}

// Function to compare permissions
function comparePermissions($current, $new) {
    $differences = [];

    // Get all sections
    $sections = array_unique(array_merge(array_keys($current), array_keys($new)));

    foreach ($sections as $section) {
        $currentSection = isset($current[$section]) ? $current[$section] : [];
        $newSection = isset($new[$section]) ? $new[$section] : [];

        // Get all keys in this section
        $keys = array_unique(array_merge(array_keys($currentSection), array_keys($newSection)));

        foreach ($keys as $key) {
            $currentValues = isset($currentSection[$key]) ? $currentSection[$key] : [];
            $newValues = isset($newSection[$key]) ? $newSection[$key] : [];

            // Compare values
            $added = array_diff($newValues, $currentValues);
            $removed = array_diff($currentValues, $newValues);

            if (!empty($added) || !empty($removed)) {
                $differences[$section][$key] = [
                    'added' => $added,
                    'removed' => $removed,
                ];
            }
        }
    }

    return $differences;
}

//Dakusan: Added this function and wrapped other echo statements into it after the fact
//Function to output a line with appId (if given), colored red (IsError=true) or green (IsError=false)
function outputColor($appId, $str, $isError=true)
{
    //Determine if coloring should be used
    static $hasColors=null;
    if(!isset($hasColors)) {
        $hasColors=stream_isatty(STDOUT) && (!trim(`command -v tput`) || intval(`tput colors`)>=16);
    }

    echo
        (!$hasColors ? '' : ($isError ? "\e[31m" : "\e[32m")).
        ($appId ? "$appId: " : '').
        $str.
        (!$hasColors ? '' : "\e[0m").
	"\n";
}

// Function to display permission differences
function displayDifferences($appId, $differences) {
    outputColor($appId, 'Permission changes:');
    foreach ($differences as $section => $keys) {
        outputColor(null, "  [$section]");
        foreach ($keys as $key => $changes) {
            if (!empty($changes['added'])) {
                outputColor(null, "    + $key = " . implode(';', $changes['added']));
            }
            if (!empty($changes['removed'])) {
                outputColor(null, "    - $key = " . implode(';', $changes['removed']));
            }
        }
    }
    echo "\n";
}

// Get architecture (moved above the loop)
$archCommand = 'flatpak --default-arch';
exec($archCommand, $archOutput);
$arch = trim(implode('', $archOutput));

// Step 1: Get the list of installed Flatpaks
$installedCommand = 'flatpak list -a --columns=application,branch,origin,options'; //Dakusan: Added -a after the fact
exec($installedCommand, $installedOutput);

// Remove header line if present
if (!empty($installedOutput) && strpos($installedOutput[0], 'Application ID') !== false) {
    array_shift($installedOutput);
}

// Build an associative array of installed applications
$installedApps = [];

foreach ($installedOutput as $line) {
    // The output is tab-delimited
    $columns = explode("\t", trim($line));
    if (count($columns) >= 4) {
        $appId = $columns[0];
        $branch = $columns[1];
        $origin = $columns[2];
        $options = $columns[3];

        $installedApps[$appId] = [
            'appId' => $appId,
            'branch' => $branch,
            'origin' => $origin,
            'options' => $options,
        ];
    }
}

// Get the list of available updates
$updatesCommand = 'flatpak remote-ls --updates --columns=application';
exec($updatesCommand, $updatesOutput);

// Remove header line if present
if (!empty($updatesOutput) && strpos($updatesOutput[0], 'Application ID') !== false) {
    array_shift($updatesOutput);
}

// Build a list of applications that have updates
$updatesAvailable = array_map('trim', $updatesOutput);

if (empty($updatesAvailable)) {
    echo "No updates available for installed Flatpaks.\n";
    exit(0);
}

$permissionChanges = [];

foreach ($updatesAvailable as $appId) {
    if (!isset($installedApps[$appId])) {
        outputColor($appId, 'Installed app not found. Skipping.'); //Dakusan: Added this line after the fact
        continue;
    }

    $app = $installedApps[$appId];
    $branch = $app['branch'];
    $origin = $app['origin'];
    $options = $app['options'];

    // Determine if it's an app or runtime
    $isRuntime = strpos($options, 'runtime') !== false;

    // Paths to the metadata files
    if ($isRuntime) {
        $metadataPath = "/var/lib/flatpak/runtime/$appId/$arch/$branch/active/metadata";
    } else {
        $metadataPath = "/var/lib/flatpak/app/$appId/$arch/$branch/active/metadata";
    }

    // Check if the metadata file exists
    if (!file_exists($metadataPath)) {
        outputColor($appId, 'Metadata file not found. Skipping.');
        continue;
    }

    // Read current permissions from the metadata file
    $metadataContent = file_get_contents($metadataPath);
    $currentPermissions = parsePermissions($metadataContent);

    // Get new metadata from remote
    $ref = $appId . '/' . $arch . '/' . $branch;
    $remoteInfoCommand = 'flatpak remote-info --show-metadata ' . escapeshellarg($origin) . ' ' . escapeshellarg($ref);

    // Clear $remoteOutput before exec()
    $remoteOutput = [];
    exec($remoteInfoCommand, $remoteOutput);

    if (empty($remoteOutput)) {
        outputColor($appId, 'Failed to retrieve remote metadata. Skipping.');
        continue;
    }

    // Parse new permissions from the remote metadata
    $newPermissions = parsePermissions($remoteOutput);

    // Compare permissions
    $differences = comparePermissions($currentPermissions, $newPermissions);

    if (!empty($differences)) {
        $permissionChanges[$appId] = $differences;
        displayDifferences($appId, $differences);
    } else { //Dakusan: Added this condition after the fact
        outputColor($appId, 'No permission changes found.', false);
    }
}

// If there are no permission changes, inform the user
if (empty($permissionChanges)) {
    echo "No permission changes detected in the available updates.\n";
}

// Ask user if they want to proceed
echo "Do you want to proceed with the updates? (y/N): ";
$handle = fopen("php://stdin", "r");
$line = fgets($handle);
$answer = trim(strtolower($line));

if ($answer == 'y' || $answer == 'yes') {
    // Proceed with updates
    echo "Updating Flatpaks...\n";
    passthru('flatpak update -y');
} else {
    echo "Updates canceled.\n";
}

fclose($handle);
?>

I created this script because I wasn’t happy with the native Roboform importer inside Bitwarden. This fixed multiple problems including:
* Ignoring MatchUrls
* Parent folders weren’t created if they had no items in them (e.x. if I had identities in Financial/Banks, but nothing in /Financial, then it wouldn’t be created /Financial)
* Completely ignoring the extra fields (RfFieldsV2)


This fixes all those problems.


This needs to be ran via php in a command line interface, so `php convert.php`. It requires the [bitwarden cli “bw” command](https://bitwarden.com/help/cli/).
There are 2 optional arguments you can pass:
1) The name of the import file. If not given, `./RfExport.csv` will be used
2) The “bw” session token. If not given as a parameter, it can be set directly inside this file on line #4. You get this token by running `bw unlock` (after logging in with `bw login`).


This script does the following:
* Reads from a csv file exported by Roboform to import into bitwarden
* Runs `bw sync` before processing
* Imported Fields:
  * Name: Becomes the name of the item
  * Url: Becomes the URL for the item
  * MatchUrl: If this is not the same “Url” then it is added as another URL for the item
  * Login: Becomes the login (user) name
  * Pwd: Becomes the password
  * Note: Becomes the note
  * Folder: Item is put in this folder
  * RfFieldsV2 (multiple):
    * Fields that match the login or password fields are marked as “linked fields” to those. If the field names are the following, they are not added as linked fields: *user, username, login, email, userid, user id, user id$, user_email, login_email, user_login, password, passwd, password$, pass, user_password, login_password, pwd, loginpassword*
    * Fields that are different than the login or password are added as appropriate “Custom fields”. Supported types: '', rad, sel, txt, pwd, rck, chk, are
    * Each field has 5 values within it, and 3 of those seem to be different types of names. So as long as they are not duplicates of each other, each name is stored as a seperate field.
* If all fields are blank but “name” and “note” then the item is considered a “Secure Note”
* While reading the csv file for import, errors and warnings are sent to stdout
* After the csv import has complete, it will give a total number of warnings/errors and ask the user if they want to continue
* Creates missing folders (including parents that have no items in them)
* During export to bitwarden:
  * This process can be quite slow since each instance of running “bw” has to do a lot of work
  * Keeps an active count of items processed and the total number of items
  * If duplicates are found (same item name and folder) then the user is prompted for what they want to do. Which is either s=skip or o=overwrite. A capitol of those letters can be given to perform the same action on all subsequent duplicates.
  * Errors are sent to stdout

<?php
global $BW_SESSION;
/** @noinspection SpellCheckingInspection,RedundantSuppression */
$BW_SESSION=($argv[2] ?? 'FILL_ME_IN');

print RunMe($argv)."\n";

//Make sure “bw” is installed, given session is valid, and is synced
function CheckBWStatus(): ?string
{
	if(is_string($StatusCheck=RunBW('status')))
		return $StatusCheck;
	else if(!is_object($StatusCheck) || !isset($StatusCheck->status))
		return 'bw return does not have status';
	else if($StatusCheck->status==='locked')
		return 'bw is locked';
	else if($StatusCheck->status!=='unlocked')
		return 'bw status is invalid';

	$ExpectedMessage='Syncing complete.';
	if(is_string($SyncCheck=RunBW('sync', null, false)))
		return $SyncCheck;
	else if($SyncCheck[0]!==$ExpectedMessage)
		return "Sync expected “${ExpectedMessage}” but got “${SyncCheck[0]}”";

	return null;
}

//Pull the known folders
function PullKnownFolders(): string|array
{
	$FolderIDs=[];
	if(is_string($FolderList=RunBW('list folders')))
		return $FolderList;
	foreach($FolderList as $Folder)
		$FolderIDs[$Folder->name]=$Folder->id;
	unset($FolderIDs['No Folder']);
	return $FolderIDs;
}

//Get the file handle and the column indexes
function GetFileAndColumns(): string|array
{
	//Prepare the import file for reading
	ini_set('default_charset', 'UTF-8');
	$FileName=$argv[1] ?? './RfExport.csv';
	if(!file_exists($FileName))
		return 'File not found: '.$FileName;
	else if(!($f=fopen($FileName, 'r')))
		return 'Error opening file: '.$FileName;

	//Check the header row values
	if(!($HeadRow=fgetcsv($f)))
		return 'Error opening file: '.$FileName;
	else if(!count($HeadRow) || $HeadRow[0]===NULL)
		return 'Missing head row: '.$FileName;
	if(str_starts_with($HeadRow[0], "\xEF\xBB\xBF")) //Remove UTF8 BOM
		$HeadRow[0]=substr($HeadRow[0], 3);
	unset($FileName);

	$ExpectedCols=array_flip(['Url', 'Name', 'MatchUrl', 'Login', 'Pwd', 'Note', 'Folder', 'RfFieldsV2']);
	$ColNums=[];
	foreach($HeadRow as $Index => $Name) {
		if(!isset($ExpectedCols[$Name]))
			if(isset($ColNums[$Name]))
				return 'Duplicate column title: '.$Name;
			else
				return 'Unknown column title: '.$Name;
		$ColNums[$Name]=$Index;
		unset($ExpectedCols[$Name]);
	}
	if(count($ExpectedCols))
		return 'Required columns not found: '.implode(', ', array_keys($ExpectedCols));
	else if($ColNums['RfFieldsV2']!==count($ColNums)-1)
		return 'RfFieldsV2 must be the last column';

	return [$f, $ColNums];
}

//Process the rows
function ProcessRows($f, array $ColNums, array &$FolderIDs): array
{
	$Counts=['Error'=>0, 'Warning'=>0, 'Success'=>0];
	$RowNum=0;
	$FolderNames=[];
	$Items=[];
	while($Line=fgetcsv($f, null, ",", "\"", "")) {
		//Process the row and add result type to counts
		$RowNum++;
		$Result=ProcessRow($Line, $ColNums);
		$Counts[$Result[0]]++;

		//Handle errors and warnings
		if($Result[0]!=='Success') {
				print "$Result[0]: Row #$RowNum $Result[1]\n";
				continue;
		} else if(isset($Result['Warnings']) && count($Result['Warnings'])) {
			print "Warning(s): Row #$RowNum ".implode("\n", $Result['Warnings'])."\n";
			$Counts['Warning']+=count($Result['Warnings']);
		}

		//Add the folder to the list of folders (strip leading slash)
		$FolderName=($Line[$ColNums['Folder']] ?? '');
		$FolderName=substr($FolderName, ($FolderName[0] ?? '')==='/' ? 1 : 0);
		$Result['Data']->folderId=&$FolderIDs[$FolderName];
		$FolderNames[$FolderName]=1;

		//Save the entry
		$Items[]=$Result['Data'];
	}
	fclose($f);
	return [$Counts, $FolderNames, $Items, $RowNum];
}

//Process a single row
function ProcessRow($Line, $ColNums): array
{
	//Skip blank lines
	if(!count($Line) || (count($Line)===1 && $Line[0]===null))
		return ['Warning', 'is blank'];

	//Extract the columns by name
	$C=(object)[];
	foreach($ColNums as $Name => $ColNum)
		$C->$Name=$Line[$ColNum] ?? '';

	//Check for errors and end processing early for notes
	if($C->Name==='')
		return ['Error', 'is missing a name'];
	else if($C->Url==='' && $C->MatchUrl==='' & $C->Login==='' && $C->Pwd==='') {
		return ['Success', 'Data'=>(object)['type'=>2, 'notes'=>$C->Note, 'name'=>$C->Name, 'secureNote'=>['type'=>0]]];
	}

	//Create a login card
	$Ret=(object)[
		'type'=>1,
		'name'=>$C->Name,
		'notes'=>$C->Note===''  ? null : $C->Note,
		'login'=>(object)[
			'uris'=>[(object)['uri'=>$C->Url]],
			'username'=>$C->Login,
			'password'=>$C->Pwd,
		],
	];
	if($C->MatchUrl!==$C->Url)
		$Ret->login->uris[]=(object)['uri'=>$C->MatchUrl];

	//Create error string for mist fields
	$i=0; //Declared up here so it can be used in $FormatError
	$FormatError=function($Format, ...$Args) use ($ColNums, &$i): string {
		return 'RfFieldsV2 #'.($i-$ColNums['RfFieldsV2']+1).' '.sprintf($Format, ...$Args);
	};

	//Handle misc/extra (RfFieldsV2) fields
	$Warnings=[];
	for($i=$ColNums['RfFieldsV2']; $i<count($Line); $i++) {
		//Pull in the parts of the misc field
		if(count($Parts=str_getcsv($Line[$i], ",", "\"", ""))!==5)
			return ['Error', $FormatError('is not in the correct format')];
		$Parts=(object)array_combine(['Name', 'Name3', 'Name2', 'Type', 'Value'], $Parts);
		if(!trim($Parts->Name))
			return ['Error', $FormatError('invalid blank name found')];

		//Figure out which “Names” to process
		$PartNames=[trim($Parts->Name)];
		foreach([$Parts->Name2, $Parts->Name3] as $NewName)
			if($NewName!=='' && !in_array($NewName, $PartNames))
				$PartNames[]=$NewName;

		//Process the different names
		foreach($PartNames as $PartName) {
			//Determined values for the item
			$LinkedID=null;
			$PartValue=$Parts->Value;
			/** @noinspection PhpUnusedLocalVariableInspection */ $Type=null; //Overwritten in all paths

			//Handle duplicate usernames and password fields
			if(($IsLogin=($PartValue===$C->Login)) || $PartValue===$C->Pwd) {
				if($Parts->Type!==($IsLogin ? 'txt' : 'pwd'))
					$Warnings[]=$FormatError('expected type “%s” but got “%s”', $IsLogin ? 'txt' : 'pwd', $Parts->Type);
				$Type=3;
				$PartValue=null;
				$LinkedID=($IsLogin ? 100 : 101);

				//If a common name then do not add the linked field
				/** @noinspection SpellCheckingInspection */
				if(in_array(
					strToLower($PartName),
					$IsLogin ?
						['user', 'username', 'login', 'email', 'userid', 'user id', 'user id$', 'user_email', 'login_email', 'user_login'] :
						['password', 'passwd', 'password$', 'pass', 'user_password', 'login_password', 'pwd', 'loginpassword']
				))
					continue;
			} else {
				//Convert the type
				switch($Parts->Type) {
					case '': //For some reason some text fields have no type given
					case 'rad':
					case 'sel':
					case 'txt': $Type=0; break;
					case 'pwd': $Type=1; break;
					case 'rck': //Radio check?
						$Type=0;
						if($PartName!==$Parts->Name) //Ignore second names for radio checks
							continue 2;
						if(count($RadioParts=explode(':', $PartName))!==2)
							return ['Error', $FormatError('radio name needs 2 parts separated by a colon')];
						$PartName=$RadioParts[0];
						$PartValue=$RadioParts[1];
						break;
					case 'chk':
						$Type=2;
						if($PartValue==='*' || $PartValue==='1')
							$PartValue='true';
						else if($PartValue==='0')
							$PartValue='false';
						else
							return ['Error', $FormatError('invalid value for chk type “%s”', $PartValue)];
						break;
					case 'are': //This seems to be a captcha
						continue 2;
					default:
						return ['Error', $FormatError('invalid field type “%s”', $Parts->Type)];
				}
			}

			//Create the return object
			if(!isset($Ret->fields))
				$Ret->fields=[];
			$Ret->fields[]=(object)[
				'name'=>$PartName,
				'value'=>$PartValue,
				'type'=>$Type,
				'linkedId'=>$LinkedID,
			];
		}
	}

	//Return finished item and warnings
	if(count($Warnings))
		$Warnings[0]="RfFieldsV2 warnings:\n".$Warnings[0];
	return ['Success', 'Data'=>$Ret, 'Warnings'=>$Warnings];
}

//Ask the user if they want to continue
function ConfirmContinue($Counts, $RowNum): ?string
{
	//Ask the user if they want to continue
	printf("Import from spreadsheet is finished. Total: %d; Successful: %d, Errors: %d, Warnings: %d\n", $RowNum, $Counts['Success'], $Counts['Error'], $Counts['Warning']);
	while(1) {
		print 'Do you wish to continue the export to bitwarden? (y/n): ';
		fflush(STDOUT);
		switch(trim(strtolower(fgets(STDIN)))) {
			case 'n':
				return 'Exiting';
			case 'y':
				break 2;
		}
	}

	return null;
}

//Get folder IDs and create parent folders for children
function GetFolders($FolderNames, &$FolderIDs): ?string
{
	foreach(array_keys($FolderNames) as $FolderName) {
		//Skip “No Folder”
		if($FolderName==='') {
			$FolderIDs[$FolderName]='';
			continue;
		}

		//Check each part of the folder tree to make sure it exists
		$FolderParts=explode('/', $FolderName);
		$CurPath='';
		foreach($FolderParts as $Index => $FolderPart) {
			$CurPath.=($Index>0 ? '/' : '').$FolderPart;

			//If folder is already cached then nothing to do
			if(isset($FolderIDs[$CurPath])) {
				continue;
			}

			//Create the folder
			print "Creating folder: $CurPath\n";
			if(is_string($FolderInfo=RunBW('create folder '.base64_encode(json_encode(['name'=>$CurPath])), 'create folder: '.$CurPath)))
				return $FolderInfo;
			else if (!isset($FolderInfo->id))
				return "bw folder create failed for “${CurPath}”: Return did not contain id";
			$FolderIDs[$CurPath]=$FolderInfo->id;
		}
	}
	return null;
}

//Pull the known items
function PullKnownItems($FoldersByID): string|array
{
	$CurItems=[];
	if(is_string($ItemList=RunBW('list items')))
		return $ItemList;
	foreach($ItemList as $Item) {
		$CurItems[($Item->folderId===null ? '' : $FoldersByID[$Item->folderId].'/').$Item->name]=$Item->id;
	}
	return $CurItems;
}

function StoreItems($Items, $CurItems, $FoldersByID): int
{
	print "\n"; //Give an extra newline before starting the processing
	$NumItems=count($Items);
	$NumErrors=0;
	$HandleDuplicatesAction=''; //The "always do this" action
	foreach($Items as $Index => $Item) {
		//Clear the current line and print the processing status
		printf("\rProcessing item #%d/%d", $Index+1, $NumItems);
		fflush(STDOUT);

		//If the item already exists then request what to do
		$FullItemName=($Item->folderId===null ? '' : $FoldersByID[$Item->folderId].'/').$Item->name;
		if(isset($CurItems[$FullItemName])) {
			//If no "always" action has been set in $HandleDuplicatesAction then ask the user what to do
			if(!($Action=$HandleDuplicatesAction)) {
				print "\n";
				while(1) {
					print "A duplicate at “${FullItemName}” was found. Choose your action (s=skip,o=overwrite,S=always skip,O=always overwrite):";
					fflush(STDOUT);
					$Val=trim(fgets(STDIN));
					if(in_array(strtolower($Val), ['s', 'o']))
						break;
				}
				$Action=strtolower($Val);
				if($Val!==$Action) //Upper case sets the "always" state
					$HandleDuplicatesAction=$Action;
			}

			//Skip the item
			if($Action==='s')
				continue;

			//Overwrite the item
			if(is_string($Ret=RunBW('edit item '.$CurItems[$FullItemName].' '.base64_encode(json_encode($Item)), 'edit item: '.$FullItemName))) {
				$NumErrors++;
				printf("\nError on item #%d: %s\n", $Index+1, $Ret);
			}
			continue;
		}

		//Create the item
		if(is_string($Ret=RunBW('create item '.base64_encode(json_encode($Item)), 'create item: '.$FullItemName))) {
			$NumErrors++;
			printf("\nError on item #%d: %s\n", $Index+1, $Ret);
		}
	}
	return $NumErrors;
}

//Run a command through the “bw” application
function RunBW($Command, $Label=null, $DecodeJSON=true): string|object|array
{
	//$Label is set to $Command if not given
	if($Label===null)
		$Label=$Command;

	//Run the command and check the results
	global $BW_SESSION;
	exec(sprintf('BW_SESSION=%s bw %s --pretty 2>&1', $BW_SESSION, $Command), $Output, $ResultCode);
	if($ResultCode===127)
		return 'bw is not installed';
	else if($ResultCode===1)
		return "bw “${Label}” threw an error: \n".implode("\n", $Output);
	else if($ResultCode!==0)
		return "bw “${Label}” returned an invalid status code [$ResultCode]: \n".implode("\n", $Output);
	else if($Output[0]==='mac failed.')
		return 'Invalid session ID';
	else if(!$DecodeJSON)
		return [implode("\n", $Output)];
	else if(($JsonRet=json_decode(implode("\n", $Output)))===null)
		return "bw “${Label}” returned non-json result";

	//Return the json object
	return $JsonRet;
}

function RunMe($argv): string
{
	//Make sure “bw” is installed and given session is valid
	if(is_string($Ret=CheckBWStatus()))
		return $Ret;

	//Pull the known folders
	if(is_string($FolderIDs=PullKnownFolders()))
		return $FolderIDs;

	//Get the file handle and the column indexes
	if(is_string($Ret=GetFileAndColumns()))
		return $Ret;
	[$f, $ColNums]=$Ret;
	unset($argv);

	//Process the rows and ask the user if they want to continue
	[$Counts, $FolderNames, $Items, $RowNum]=ProcessRows($f, $ColNums, $FolderIDs);
	if(is_string($Ret=ConfirmContinue($Counts, $RowNum)))
		return $Ret;
	unset($Counts, $RowNum, $f, $ColNums);

	//Get folder IDs and create parent folders for children
	if(is_string($Ret=GetFolders($FolderNames, $FolderIDs)))
		return $Ret;
	unset($FolderNames);

	//Pull the known items
	$FoldersByID=array_flip($FolderIDs);
	if(is_string($CurItems=PullKnownItems($FoldersByID)))
		return $CurItems;

	//Store all items
	$FolderIDs['']=null; //Change empty folder to id=null for json insertions
	$NumErrors=StoreItems($Items, $CurItems, $FoldersByID);

	//Return completion information
	return "\nCompleted ".($NumErrors ? "with $NumErrors error(s)" : 'successfully');
}

#This script takes a newline delimited file list from STDIN for md5 hashing
#This script requires the `md5sum`, `pv`, `paste`, `bc`, and 'numfmt' commands

#The output of the md5s are stored in the file specified by the first parameter
#The format for each md5 hash to the output file is "$FileName\t$Hash\n"

#File sizes are always output in megabytes with 3 decimal places
#While calculating the hashes the script keeps the user informed of the progress of both the current file and all the files as follows:
#1) Before file starts: "Hashing: $FileName ($FileSize MiB)\n"
#2) During transfer: The progress of the hash of the current file ran through `pv`
#3) During transfer: The progress of the hashing of all the files, ran through `pv`
#4) After transfer: "Finished $TotalProgressPercent% ($ProcessedBytes/$TotalBytes MiB)\n\n"

#Get $Outfile from the first argument and the $FileList from STDIN (newline delimited)
OutFile="$1";
FileList=`cat /dev/stdin`

#Format a byte count in MegaBytes with comma grouping and 3 decimal places
MbFmtNoExt ()
{
	echo "scale=3; $1/1024/1024" | bc | echo -n `xargs numfmt --grouping`
}

#Add " MiB" to the end of MbFmtNoExt
MbFmt ()
{
	echo `MbFmtNoExt $1`" MiB"
}

#Calculate and output the total size of the file list
echo -n "Calculating total size: "
TotalSize=`echo "$FileList" | xargs -d"\n" stat --printf="%s\n" | paste -s -d+ | bc`
MbFmt $TotalSize
echo #Add an extra newline

#Create a fifo to keep track of the total complete
TotalDoneFifo=$(mktemp)
TotalDoneBG=0
rm "$TotalDoneFifo"
mkfifo "$TotalDoneFifo"
cat > "$TotalDoneFifo" & #Do not close read of fifo
Cleanup() {
	rm "$TotalDoneFifo"
	kill $TotalDoneBG
	exit 0
}
trap Cleanup SIGTERM SIGINT

#Start the TOTAL line
tail -f "$TotalDoneFifo" | pv -s $TotalSize -F  "%b %t %p %e" > /dev/null &
TotalDoneBG=$!

#Run over the list (newline delimited)
CalculatedBytes=0
IFS=$'\n'
for FileName in `echo "$FileList"`
do
	#Output the file size and name to STDOUT
	FileSize=`stat --printf="%s" "$FileName"`
	echo "Hashing: $FileName ("`MbFmt $FileSize`")"

	#Output the filename to $OutFile
	echo -n $FileName$'\t' >> $OutFile

	#Run the md5 calculation with `pv` progress
	#Output the hash to $OutFile after the FileName and a tab
	cat "$FileName" | pv -s $FileSize -c | tee -a "$TotalDoneFifo" | md5sum | awk '{print $1}' >> $OutFile

	#Output the current progress for the entire file list
	#Format: "Finished $TotalProgressPercent% ($ProcessedBytes/$TotalBytes MiB)\n\n"
	CalculatedBytes=$(($CalculatedBytes+$FileSize))
	echo -n "Finished "
	printf "%.3f" `echo "scale=4; $CalculatedBytes*100/$TotalSize" | bc`
	echo "% ("`MbFmtNoExt $CalculatedBytes`"/"`MbFmt $TotalSize`$')\n'
done

Cleanup

The following is a simple bash script to ping a different domain once a second and log the output. By default, it pings #.castledragmire.com, where # is an incrementing number starting from 0.

The script is written for Cygwin (See the PING_COMMAND variable at the top) but is very easily adaptable to Linux.

The log output is: EPOCH_TIMESTAMP DOMAIN PING_OUTPUT

#This uses Window's native ping since the Cygwin ping is sorely lacking in options
#"-n 1"=Only runs once, "-w 3000"=Timeout after 3 seconds
#The grep strings are also directly tailored for Window's native ping
PING_COMMAND=$(
	echo 'C:/Windows/System32/PING.EXE -n 1 -w 3000 $DOMAIN |';
	echo 'grep -iP "^(Request timed out|Reply from|Ping request could not find)"';
)

i=0 #The subdomain counter
STARTTIME=`date +%s.%N` #This holds the timestamp of the end of the previous loop

#Infinite loop
while true
do
	#Get the domain to run. This requires a domain that has a wildcard as a primary subdomain
	DOMAIN="$i.castledragmire.com"

	#Output the time, domain name, and ping output
	echo `date +%s` "$DOMAIN" $(eval $PING_COMMAND)

	#If less than a second has passed, sleep up to 1 second
	ENDTIME=`date +%s.%N`
	SLEEPTIME=$(echo "1 - ($ENDTIME - $STARTTIME)" | bc)
	STARTTIME=$ENDTIME
	if [ $(echo "$SLEEPTIME>0" | bc) -eq 1 ]; then
		sleep $SLEEPTIME
		STARTTIME=$(echo "$STARTTIME + $SLEEPTIME" | bc)
	fi

	#Increment the subdomain counter
	let i+=1
done

Following is some C++ source code for a Windows kernel-driver service loader. It could be used to load other service types too by changing the dwServiceType flag on the CreateService call. I threw this together for another project I am currently working on. It is also used in the following post (posting soon).

• It is a command line utility which takes 3 arguments:
  1. The service name. Hereby referred to as SERVICE_NAME
  2. The service display name. Hereby referred to as DISPLAY_NAME
  3. The driver path (to the .sys file). Hereby referred to as DRIVER_PATH
• This program (most likely) requires administrative access. There are also some caveats regarding driver code signing requirements that are thoroughly explored elsewhere.
• It first checks to see if a service already exists with the given SERVICE_NAME. If it does:
  1. If the DISPLAY_NAME matches, the service is kept as is.
  2. If the DISPLAY_NAME does not match, the user is prompted on if they want to delete the current service. If they do not, the program exits.
• If the service needs to be created (it did not already exist or was deleted), it creates the service with the given SERVICE_NAME, DISPLAY_NAME, and DRIVER_PATH. If the service is not created during this run, the DRIVER_PATH is ignored.
  Note: The DRIVER_PATH must be to a direct local file system file. I have found that network links and symbolic links do not work.
• The service is started up:
  • If it is already running, the user is prompted on if they want to stop the currently running service. If they say no, the program exits.
• The program then waits for a final user input on if they want to close the service before exiting the program.
• If there was an error, the program reports the error, otherwise, it reports “Success”.
• The program pauses at the end until the user presses any key to exit.
• The program returns 0 on success, and 1 if an error occurred.

//Compiler flags
#define WIN32_LEAN_AND_MEAN  //Include minimum amount of windows stuff
#ifndef _UNICODE //Everything in this script is unicode
	#define _UNICODE
#endif

#include <windows.h>
#include <stdio.h>
#include <conio.h>
#include <memory>

//Smart pointers
typedef std::unique_ptr<WCHAR, void(*)(WCHAR*)> SmartWinAlloc;
typedef std::unique_ptr<SC_HANDLE, void(*)(SC_HANDLE*)> SmartCloseService;
void Delete_SmartWinAlloc(WCHAR *p) { if(p) LocalFree(p); }
void Delete_SmartCloseService(SC_HANDLE *h) { if(h && *h) CloseServiceHandle(*h); }

//Function declarations
WCHAR* InitDriver(int argc, WCHAR *argv[]);
WCHAR* FormatError(WCHAR* Format, ...);
SmartWinAlloc GetLastErrorStr();
BOOLEAN AskQuestion(WCHAR* Question); //Returns if user answered yes

int wmain(int argc, WCHAR *argv[])
{
	//Run the init routine
	WCHAR* Ret=InitDriver(argc, argv);

	//If there is an error, report it, or otherwise, report success
	wprintf(L"%s\n", Ret ? Ret : L"Success");
	wprintf(L"%s\n", L"Press any key to exit");
	_getch();

	//Return if successful
	return (Ret ? 1 : 0);
}

WCHAR* InitDriver(int argc, WCHAR *argv[])
{
	//Confirm arguments
	if(argc<4)
		return FormatError(L"%s", L"3 arguments are required: Service Name, Display Name, Driver Path");
	const WCHAR* Param_ServiceName=argv[1];
	const WCHAR* Param_DisplayName=argv[2];
	const WCHAR* Param_DriverPath =argv[3];

	//Open the service manager
	wprintf(L"%s\n", L"Opening the service manager");
	SC_HANDLE HSCManager=OpenSCManager(nullptr, nullptr, SC_MANAGER_CREATE_SERVICE);
	if(!HSCManager)
		return FormatError(L"%s: %s", L"Error opening service manager", GetLastErrorStr());
	SmartCloseService FreeHSCManager(&HSCManager, Delete_SmartCloseService);

	//Check if the service already exists
	wprintf(L"%s\n", L"Checking previously existing service state");
	BOOL ServiceExists=false;
	{
		//Get the service name
		const DWORD NameBufferSize=255;
		WCHAR NameBuffer[NameBufferSize];
		WCHAR *NamePointer=NameBuffer;
		DWORD NamePointerSize=NameBufferSize;
		std::unique_ptr<WCHAR> Buf(nullptr); //May be swapped with a real pointer later
		for(INT_PTR i=0;i<2;i++)
		{
			//If we found the service, exit the lookup here
			if(GetServiceDisplayName(HSCManager, Param_ServiceName, NamePointer, &NamePointerSize))
			{
				ServiceExists=true;
				break;
			}

			//If the service does not exist, we can exit the lookup here
			if(GetLastError()==ERROR_SERVICE_DOES_NOT_EXIST)
				break;

			//If error is not insufficient buffer size, return the error
			if(GetLastError()!=ERROR_INSUFFICIENT_BUFFER)
				return FormatError(L"%s: %s", L"Could not query service information", GetLastErrorStr());

			//If second pass, error out
			if(i==1)
				return FormatError(L"%s: %s", L"Could not query service information", L"Second buffer pass failed");

			//Create a buffer of appropriate size (and make sure it will later be released)
			NamePointer=new WCHAR[++NamePointerSize];
			std::unique_ptr<WCHAR> Buf2(NamePointer);
			Buf.swap(Buf2);
		}

		//If the service already exists, confirm the service name matches, and if not, ask if user wants to delete the current service
		if(ServiceExists)
		{
			wprintf(L"%s\n", L"The service already exists");
			if(wcsncmp(NamePointer, Param_DisplayName, NamePointerSize+1))
			{
				//If the server names do not match, ask the user what to do
				wprintf(L"%s:\nCurrent: %s\nRequested: %s\n", L"The service names do not match", NamePointer, Param_DisplayName);

				//Make the request
				if(!AskQuestion(L"Would you like to replace the service? (y/n)")) //If user does not wish to replace the service
					return FormatError(L"%s", L"Cannot continue if service names do not match");

				//Delete the service
				wprintf(L"%s\n", L"Deleting the old service");
				ServiceExists=false;
				SC_HANDLE TheService=OpenService(HSCManager, Param_ServiceName, DELETE);
				if(!TheService)
					return FormatError(L"%s: %s", L"Could not open the service to delete it", GetLastErrorStr());
				SmartCloseService CloseTheService(&TheService, Delete_SmartCloseService); //Close the service handle
				if(!DeleteService(TheService))
					return FormatError(L"%s: %s", L"Could not delete the service", GetLastErrorStr());
				wprintf(L"%s\n", L"The service has been deleted");
			}
		}
	}

	//Create the service
	SC_HANDLE TheService;
	if(!ServiceExists)
	{
		//Confirm the driver path exists
		wprintf(L"%s\n", L"Checking the driver file");
		DWORD FileAttrs=GetFileAttributes(Param_DriverPath);
		if(FileAttrs==INVALID_FILE_ATTRIBUTES)
			return FormatError(L"%s: %s", L"Given path is invalid", GetLastErrorStr());
		if(FileAttrs&FILE_ATTRIBUTE_DIRECTORY)
			return FormatError(L"%s: %s", L"Given path is invalid", L"Path is a folder");

		//Create the service
		wprintf(L"%s\n", L"Creating the service");
		TheService=CreateService(
			HSCManager, Param_ServiceName, Param_DisplayName, 
			SERVICE_START|SERVICE_STOP, 
			SERVICE_KERNEL_DRIVER, SERVICE_DEMAND_START, SERVICE_ERROR_IGNORE,
			Param_DriverPath, nullptr, nullptr, nullptr, nullptr, nullptr);
		if(!TheService)
			return FormatError(L"%s: %s", L"Could not create the service", GetLastErrorStr());

	//Open the service if not creating
	} else {
		TheService=OpenService(HSCManager, Param_ServiceName, SERVICE_START|SERVICE_STOP);
		if(!TheService)
			return FormatError(L"%s: %s", L"Could not open the service", GetLastErrorStr());
	}
	SmartCloseService CloseTheService(&TheService, Delete_SmartCloseService); //Close the service on exit

	//Start the service
	wprintf(L"%s\n", L"Starting the service");
	for(INT_PTR i=0;i<2;i++)
	{
		if(StartService(TheService, 0, nullptr))
			break;

		//If not "service already running" error, or user does not want to stop the current service
		if(i==1 || GetLastError()!=ERROR_SERVICE_ALREADY_RUNNING || !AskQuestion(L"The service is already running. Would you like to stop it? (y/n)"))
			return FormatError(L"%s: %s", L"Could not start the service", GetLastErrorStr());

		//Stop the service
		SERVICE_STATUS ss;
		wprintf(L"%s\n", L"Stopping the current service");
		if(!ControlService(TheService, SERVICE_CONTROL_STOP, &ss))
			return FormatError(L"%s: %s", L"Could not stop the current service", GetLastErrorStr());
	}
	wprintf(L"%s\n", L"Started the service");

	//Ask if the user wants to close the service
	if(!AskQuestion(L"Would you like to stop the service before exit? (y/n)"))
		return nullptr;

	//Stop the service
	SERVICE_STATUS ss;
	if(!ControlService(TheService, SERVICE_CONTROL_STOP, &ss))
		return FormatError(L"%s: %s", L"Could not stop the service", GetLastErrorStr());
	if(ss.dwCurrentState!=SERVICE_STOP_PENDING && ss.dwCurrentState!=SERVICE_STOPPED)
		return FormatError(L"%s", L"The service does not appear to be closing");
	wprintf(L"%s\n", L"The service has been stopped");

	//Return success
	return nullptr;
}

WCHAR* FormatError(WCHAR* Format, ...)
{
	static WCHAR Err[255];
	va_list VAList;
	va_start(VAList, Format);
	vswprintf(Err, sizeof(Err)/sizeof(Err[0]), Format, VAList);
	return Err;
}

SmartWinAlloc GetLastErrorStr()
{
	LPWSTR MessageBuffer=nullptr;
	FormatMessage(
		FORMAT_MESSAGE_ALLOCATE_BUFFER|FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS|FORMAT_MESSAGE_MAX_WIDTH_MASK,
		nullptr, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPWSTR)&MessageBuffer, 0, nullptr);
	return SmartWinAlloc(MessageBuffer, Delete_SmartWinAlloc);
}

BOOLEAN AskQuestion(WCHAR* Question)
{
	//Make the request and wait for an input character
	while(1)
	{
		//Ask the question and get the answer
		wprintf(L"%s:", Question);
		fflush(stdout);
		char InputChar=_getch();
		printf("\n");

		//Check for a valid answer
		if(InputChar=='n' || InputChar=='N')
			return FALSE;
		if(InputChar=='y' || InputChar=='Y')
			return TRUE;
	}
}

• Takes raw input and wraps long lines to indent against the current line start.
• When an indented/list line is encountered, which starts with spaces followed by a star "*", wrapped line’s indents will start 2 spaces after the star.
• Lines attempt to split at words of 10 characters or less (see .MinCharsInSplitWord).
• If a line needs to split along a word longer than this, a hyphen is inserted at the end of the line.

import argparse
import re
class BulletHelpFormatter(argparse.HelpFormatter):
    def __init__(self, *args, **kwargs):
        super(BulletHelpFormatter, self).__init__(*args, **kwargs)
        self.MinCharsInSplitWord=10

    def _split_lines(self, text, width):
        #Split lines around line breaks and then modify each line
        Lines=[]
        for Line in text.splitlines():
            #Get the number of spaces to put at subsequent lines
            #0 if not a list item, oherwise, 2+list item start
            ListEl=re.match(r'^ *\*', Line)
            NumBeginningSpace=(0 if ListEl==None else ListEl.end()+1)

            #Add extra spaces at the beginning of each line to match the start of the current line, and go to a maxium of $width
            IsFirstPass=True
            SpacesToAdd=''
            NumSpacesToAdd=0
            while(True):
                #Get the word break points before and after where the line would end
                MaxLineLen=max(min(width-NumSpacesToAdd, len(Line)), 1)
                PrevWordBreak=CurWordBreak=0
                for WordBreak in re.finditer(r'(?<=\W).|\W|$', Line):
                    PrevWordBreak=CurWordBreak
                    CurWordBreak=WordBreak.start()
                    if CurWordBreak>=MaxLineLen:
                        if CurWordBreak==MaxLineLen:
                            PrevWordBreak=CurWordBreak
                        break

                #If previous wordbreak is more than MinCharsInSplitWord away from MaxLineLen, then split at the end of the line
                IsSplit=(PrevWordBreak<1 or CurWordBreak-PrevWordBreak>self.MinCharsInSplitWord)
                SplitPos=(MaxLineLen if IsSplit else PrevWordBreak)

                #Append the new line to the list of lines
                Lines.append(SpacesToAdd+Line[0:SplitPos]+('-' if IsSplit else ''))
                Line=Line[SplitPos:]

                #If this is the end, nothing left to do
                if len(Line)==0:
                    break

                #If this is the first pass, update line creation variables
                if IsFirstPass:
                    IsFirstPass=False
                    NumSpacesToAdd=NumBeginningSpace
                    SpacesToAdd=(' ' * NumSpacesToAdd)

        return Lines

Cygwin has had a long time problem that, depending on your configuration, may cause you to be unable to send a SIGINT (interrupt signal via Ctrl+C) to a native Windows command line executables. As a matter of fact, trying to do so may completely freeze up the console, requiring a process kill of the actual console, bash, and the executable you ran. This problem can crop up for many reasons including the version of Cygwin you are running and your terminal emulator. I specifically installed mintty as my default Cygwin console to get rid of this problem a long time ago (among many other features it had), and now it even has this problem.

While my normal solution is to try and steer clear of native Windows command line executables in Cygwin, this is not always an option. Golang was also causing me this problem every time I ran a network server, which was especially problematic as I would have to ALSO manually kill the server process or it would continue to hold the network port so another test of the code could not use it. An example piece of code is as follows:

package main
import ( "net/http"; "fmt" )
func main() {
	var HR HandleRequest
	if err := http.ListenAndServe("127.0.0.1:81", HR); err!=nil {
		fmt.Println("Error starting server") }
}

//Handle a server request
type HandleRequest struct{}
func (HR HandleRequest) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	fmt.Printf("Received connection from: %s\n", req.RemoteAddr)
}

go run example.go

go build example.go && example.exe

func ListenForExitCommand() {
	for s:=""; s!="exit"; { //Listen for the user to type exit
		if _, err:=fmt.Scanln(&s); err!=nil {
			if err.Error()!="unexpected newline" {
				fmt.Println(err) }
		} else if s=="flush" || s=="exit" {
			//Clean up everything here
		}
	}
	fmt.Println("Exit received, closing process")
	os.Exit(1)
}

go ListenForExitCommand() //Listen for "exit" command

TARGET=`echo $1 | perl -pe 's/\\//\\\\/g'`; #Determine the target from the first parameter (where we are linking to). Change forward slashes to back slashes
LINK=`echo $2 | perl -pe 's/\\//\\\\/g'` #Determine the link name from the second parameter (where the symlink is made). Change forward slashes to back slashes
cmd /c mklink $3 $4 $5 $6 "$LINK" "$TARGET" #Perform the windows mklink command with optional extra parameters

• /d : directory symlink (more hard)
• /j : directory junction (more soft)

• No longer limited to just 4 extra parameters (dynamic instead of static)
• Can now just specify your directory as the link location, and the filename will be automatically filled in
• Handles spaces better

#Get the target and link
TARGET="$1"
shift
LINK="$1"
shift

#If the link is already a directory, append the filename to the end of it
if [ -d "$LINK" ]; then
    #Get the file/directory name without the rest of the path
    ELEMENT_NAME=`echo "$TARGET" | perl -pe 's/^.*?\/([^\/]+)\/?$/$1/'`

    #Append the file name to the target, making sure there is only 1 separating "/"
    LINK=`echo "$LINK" | perl -pe 's/^(.*?)\/?$/$1/'`
    LINK="$LINK"/"$ELEMENT_NAME"
fi

#Replace forward slashes with back slashes
TARGET=`echo $TARGET | perl -pe 's/\\//\\\\/g'`
LINK=`echo $LINK | perl -pe 's/\\//\\\\/g'`

#Perform the windows mklink command with optional extra parameters
cmd /c mklink "$@" "$LINK" "$TARGET"

Rsync is a spectacular bash utility for doing file copying and/or syncing operations. It has a multitude of switches to help optimize and handle any requirements for file copy operation over a local computer or network. However, sometimes networks are less than stable and stalls can happen during an rsync (or scp). This is quite the nuisance when doing very large (i.e. GB) transfers. To solve this, the following script can be used to auto resume a stalled rsync.

export Result=1;
while [ $Result -ne 0 ]; do
  echo "STARTING ($Result) @" `date`;
  rsync -Pza --timeout=10 COPY_FROM COPY_TO_USER@COPY_TO_HOST:COPY_TO_LOCATION;
  Result=$?;
  sleep 1;
done

• The -P switch is highly suggested as it activates:
  • --partial: This keeps a file even if it doesn’t finish transferring so it can be resumed when rsync is restarted. This is especially important if you have very large files.
  • --progress: This shows you the progress of the current file copy operation.
• The -z switch turns on gzip compression during the file transfer, so may only be useful depending on the circumstances.
• The -a switch stands for “archive” and is generally a good idea to use. It includes the switches:
  • -r: Recurse into folders
  • -t: Preserves file modification time stamp. This is highly recommended for this, and incremental backups, as rsync, by default, skips files whose file sizes and modification times match.
  • -l and -D: Preserve file type (i.e. symlinks)
  • -p: Preserve file (chmod) permissions
  • -g and -o: Preserve file owners.
• The --timeout is the crux of the script, in that if an I/O timeout of 10 seconds occurs, the rsync exits prematurely so it can be restarted.

For more useful switches and information, see the rsync man page.

export Result=1; #This will hold the result of the rsync. Set to 1 so the first loop check will fail.
while [ $Result -ne 0 ]; do #Loop until rsync result is successful
  echo "STARTING ($Result) @" `date`; #Inform the user of the time an rsync is starting and the last rsync failure code
  rsync -Pza --timeout=10 COPY_FROM COPY_TO_USER@COPY_TO_HOST:COPY_TO_LOCATION; #See rest of post for switch information
  Result=$?; #Store the result of the rsync
  sleep 1; #This is an optional 1 second timeout between attempts
done

I had the need to pass a program’s [standard] output to a web browser in real time. The best solution for this is to use a combination of programs made in different languages. The following are all of these individual components to accomplish this task.

Please note the C components are only compatible with gcc and bash (cygwin required for Windows), as MSVC and Windows command prompt are missing vital functionality for this to work.

The first component is a server made in C that receives stdin (as a pipe, or typed by the user after line breaks) and sends that data out to a connected client (buffering the output until the client connects).

PassThruServer source, PassThruServer compiled Windows executable.

• This compiles as C99 under gcc:

  gcc PassThruServer.c -o PassThruServer

• Define “WINDOWS” when compiling in Windows (pass “-DWINDOWS”)

#include <stdio.h>
#include <malloc.h>
#include <fcntl.h>
#include <sys/types.h> 
#include <sys/socket.h>
#include <netinet/in.h>
#include <signal.h>

//The server socket and options
int ServerSocket=0;
const int PortNumber=1234; //The port number to listen in on

//If an error occurs, exit cleanly
int error(char *msg)
{
	//Close the socket if it is still open
	if(ServerSocket)
		close(ServerSocket);
	ServerSocket=0;

	//Output the error message, and return the exit status
	fprintf(stderr, "%s\n", msg);
	return 1;
}

//Termination signals
void TerminationSignal(int sig)
{
	error("SIGNAL causing end of process");
	_exit(sig);
}

int main(int argc, char *argv[])
{
	//Listen for termination signals
	signal(SIGINT, TerminationSignal);
	signal(SIGTERM, TerminationSignal);
	signal(SIGHUP, SIG_IGN); //We want the server to continue running if the environment is closed, so SIGHUP is ignored -- This doesn't work in Windows
	
	//Create the server
	struct sockaddr_in ServerAddr={AF_INET, htons(PortNumber), INADDR_ANY, 0}; //Address/port to listen on
	if((ServerSocket=socket(AF_INET, SOCK_STREAM, 0))<0) //Attempt to create the socket
		return error("ERROR on 'socket' call");
	if(bind(ServerSocket, (struct sockaddr*)&ServerAddr, sizeof(ServerAddr))<0) //Bind the socket to the requested address/port
		return error("ERROR on 'bind' call");
	if(listen(ServerSocket,5)<0) //Attempt to listen on the requested address/port
		return error("ERROR on 'listen' call");

	//Accept a connection from a client
	struct sockaddr_in ClientAddr;
	int ClientAddrLen=sizeof(ClientAddr);
	int ClientSocket=accept(ServerSocket, (struct sockaddr*)&ClientAddr, &ClientAddrLen);
	if(ClientSocket<0) 
		return error("ERROR on 'accept' call");

	//Prepare to receive info from STDIN
		//Create the buffer
		const int BufferSize=1024*10;
		char *Buffer=malloc(BufferSize); //Allocate a 10k buffer
		//STDIN only needs to be set to binary mode in windows
		const int STDINno=fileno(stdin);
		#ifdef WINDOWS
			_setmode(STDINno, _O_BINARY);
		#endif
		//Prepare for blocked listening (select function)
		fcntl(STDINno, F_SETFL, fcntl(STDINno, F_GETFL, 0)|O_NONBLOCK); //Set STDIN as blocking
		fd_set WaitForSTDIN;
		FD_ZERO(&WaitForSTDIN);
		FD_SET(STDINno, &WaitForSTDIN);

	//Receive information from STDIN, and pass directly to the client
	int RetVal=0;
	while(1)
	{
		//Get the next block of data from STDIN
		select(STDINno+1, &WaitForSTDIN, NULL, NULL, NULL); //Wait for data
		size_t AmountRead=fread(Buffer, 1, BufferSize, stdin); //Read the data
		if(feof(stdin) || AmountRead==0) //If input is closed, process is complete
			break;
		
		//Send the data to the client
		if(write(ClientSocket,Buffer,AmountRead)<0) //If error in network connection occurred
		{
			RetVal=error("ERROR on 'write' call");
			break;
		}
	}
	
	//Cleanup
	if(ServerSocket)
		close(ServerSocket);
	free(Buffer);
	
	return RetVal;
}

The next component is a Flash applet as the client to receive data. Flash is needed as it can keep a socket open for realtime communication. The applet receives the data and then passes it through to JavaScript for final processing.

Compiled Flash Client Applet

import flash.external.ExternalInterface;
import flash.events.Event;
ExternalInterface.addCallback("OpenSocket", OpenSocket);

function OpenSocket(IP:String, Port:Number):void
{
	SendInfoToJS("Trying to connect");
	var TheSocket:Socket = new Socket();
	TheSocket.addEventListener(Event.CONNECT, function(Success) { SendInfoToJS(Success ? "Connected!" : "Could not connect"); });
	TheSocket.addEventListener(Event.CLOSE, function() { SendInfoToJS("Connection Closed"); });
	TheSocket.addEventListener(IOErrorEvent.IO_ERROR, function() {SendInfoToJS("Could not connect");});
	TheSocket.addEventListener(ProgressEvent.SOCKET_DATA, function(event:ProgressEvent):void { ExternalInterface.call("GetPacket", TheSocket.readUTFBytes(TheSocket.bytesAvailable)); });
	TheSocket.connect(IP, Port);
}
function SendInfoToJS(str:String) { ExternalInterface.call("GetInfoFromFlash", str); }
stop();

var NewSock=new XMLSocket();
NewSock.onData=function(msg) { GetPacket(msg); }
NewSock.onConnect=function(Success) { SendInfoToJS(Success ? "Connected!" : "Could not connect"); }
SendInfoToJS(NewSock.connect(IP, Port) ? "Trying to Connect" : "Could not start connecting");

JavaScript can then receive (and send) information from (and to) the Flash applet through the following functions.

• FLASH.OpenSocket(String IP, Number Port): Call this from JavaScript to open a connection to a server. Note the IP MIGHT have to be the domain the script is running on for security errors to not be thrown.
• JAVASCRIPT.GetInfoFromFlash(String): This is called from Flash whenever connection information is updated. I have it giving arbitrary strings ATM.
• JAVASCRIPT.GetPacket(String): This is called from Flash whenever data is received through the connection.

This example allows the user to input the IP to connect to that is streaming the output. Connection information is shown in the “ConnectionInfo” DOM object. Received data packets are appended to the document in separate DOM objects.

JavaScript+HTML Source

var isIE=navigator.appName.indexOf("Microsoft")!=-1;
function getFlashMovie(movieName) { return (isIE ? window[movieName] : document[movieName]);  }
function $(s) { return document.getElementById(s); }

function Connect()
{
	getFlashMovie("client").OpenSocket($('IP').value, 1234);
}

function GetInfoFromFlash(Str)
{
	$('ConnectionInfo').firstChild.data=Str;
}

function GetPacket(Str)
{
	var NewDiv=document.createElement('DIV');
	NewDiv.appendChild(document.createTextNode(Str));
	$('Info').appendChild(NewDiv);
}

Next is an example application that outputs to stdout. It is important that it flushes stdout after every output or the communication may not be real time.

inc source, inc compiled Windows executable.

inc counts from 0 to one less than a number (parameter #1 [default=50]) after a certain millisecond interval (parameter #2 [default=500]).

./inc 10 #Counts from 0-9 every half a second

#include <stdio.h>
#include <stdlib.h>

int main(int argc, char *argv[])
{
	int NumLoops=(argc>1 ? atoi(argv[1]) : 50); //Number of loops to run from passed argument 1. Default is 50 if not specified.
	int LoopWait=(argc>2 ? atoi(argv[2]) : 500); //Number of milliseconds to wait in between each loop from passed argument 2. Default is 500ms if not specified.
	LoopWait*=1000; //Convert to microseconds for usleep

	//Output an incremented number every half a second
	int i=0;
	while(i<NumLoops)
	{
		printf("%u\n", i++);
		fflush(stdout); //Force stdout flush
		usleep(LoopWait); //Wait for half a second
	};
	
	return 0;
}

This final component is needed so the Flash applet can connect to a server. Unfortunately, new versions of Flash (at least version 10, might have been before that though) started requiring policies for socket connections >:-(. I don’t think this is a problem if you compile your applet to target an older version of Flash with the ActionScript v1.0 code.

This Perl script creates a server on port 843 to respond to Flash policy requests, telling any Flash applet from any domain to allow connections to go through to any port on the computer (IP). It requires Perl, and root privileges on Linux to bind to a port <1024 (su to root or run with sudo).

Flash Socket Policy Server (Rename extension to .pl)

#!/usr/bin/perl
use warnings;
use strict;

#Listen for kill signals
$SIG{'QUIT'}=$SIG{'INT'}=$SIG{__DIE__} = sub
{
	close Server;
	print "Socket Policy Server Ended: $_[0]\n";
	exit;
};

#Start the server:
use Socket;
use IO::Handle;
my $FlashPolicyPort=843;
socket(Server, PF_INET, SOCK_STREAM, getprotobyname('tcp')) or die "'socket' call: $!"; #Open the socket
setsockopt(Server, SOL_SOCKET, SO_REUSEADDR, 1) or die "'setsockopt' call: $!"; #Allow reusing of port/address if in TIME_WAIT state
bind(Server, sockaddr_in($FlashPolicyPort,INADDR_ANY)) or die "'bind' call: $!"; #Listen on port $FlashPolicyPort for connections from any INET adapter
listen(Server,SOMAXCONN) or die "'listen' call: $!"; #Start listening for connections
Server->autoflush(1); #Do not buffer output

#Infinite loop that accepts connections
$/ = "\0"; #Reset terminator from new line to null char
while(my $paddr=accept(Client,Server))
{
	Client->autoflush(1); #Do not buffer IO
	if(<Client> =~ /.*policy\-file.*/i) { #If client requests policy file...
		print Client '<cross-domain-policy><allow-access-from domain="*" to-ports="*" /></cross-domain-policy>'.$/; #Output policy info: Allow any flash applets from any domain to connect
	}
	close Client; #Close the client
}

1. Start the servers
   • In your [bash] command shell, execute the following

     Server/FlashSocketPolicy.pl & #Run the Flash Policy Server as a daemon. Don't forget sudo in Linux
     ./inc | ./PassThruServer #Pipe inc out to the PassThruServer

   • Note that this will immediately start the PassThruServer receiving information from “inc”, so if you don’t get the client up in time, it may already be done counting and send you all the info at once (25 seconds).
   • The PassThruServer will not end until one of the following conditions has been met:
     • The client has connected and the piped process is completed
     • The client has connected and disconnected and the disconnect has been detected (when a packet send failed)
     • It is manually killed through a signal
   • The Flash Policy Server daemon should probably just be left on indefinitely in the background (it only needs to be run once).
2. To run the client, open client.html through a web server [i.e. Apache’s httpd] in your web browser. Don’t open the local file straight through your file system, it needs to be run through a web server for Flash to work correctly.
3. Click “connect” (assuming you are running the PassThruServer already on localhost [the same computer]). You can click “connect” again every time a new PassThruServer is ran.

This is a modification of the utility I made in yesterday’s post, chunk, and most of the information mentioned in that post applies to this one too. This utility instead writes bytes from STDIN to a certain byte offset of a preexisting file. Below is the source code for the result, which I call chunkwrite (Windows Executable).

Chunkwrite writes bytes to a file at a given offset from STDIN. The parameters are:
1) The file to write to
2) The byte offset to write at (hex is supported like 0xA)

//Copyright 2009 by Dakusan (http://www.castledragmire.com/Copyright). Licensed under Dakusan License v2.0 (http://www.castledragmire.com/Misc/Software_Licenses/Dakusan_License_v2.0.php).
//See http://www.castledragmire.com/Posts/chunkwrite for more information

#define __LARGE64_FILES
#include <stdio.h>
#include <stdlib.h> //strtoull

#ifdef WIN32 //STDIN only needs to be set to binary mode in windows
	#include <io.h> //_setmode
	#include <fcntl.h> //_O_BINARY
#endif

typedef unsigned long long UINT64;
const UINT64 MaxSizeToRead=1024*1024*10; //The maximum number of bytes to read at a time to our buffer (Must be < 2^31)

UINT64 GetNumberFromString(const char* S) //Extract both hexidecimal and decimal numbers from a string
{
	bool IsHex=S[0]=='0' && (S[1]|32=='x'); //If string starts as 0x, then is a hex number
	return strtoull(S+(IsHex ? 2 : 0), NULL, IsHex ? 16 : 10); //Hex number starts after 2 characters and uses base 16
}

int main(int argc, char *argv[], char *envp[])
{
	//Determine if proper number of parameters are passed, and if not, output help info
	if(argc!=3)
		return fprintf(stderr, "Chunkwrite writes bytes to a file at a given offset from STDIN. The parameters are:\n1) The file to write to\n2) The byte offset to write at (hex is supported like 0xA)\n") & 0;

	//Open the file to output to
	FILE *TheFile=fopen64(argv[1], "r+b");
	if(TheFile==NULL)
		return fprintf(stderr, "File not found or cannot open file\n") & 0;

	//Determine the requested start offset
	UINT64 Offset=GetNumberFromString(argv[2]);

	//Write the data 10MB at a time from STDIN to the file
	char *Buffer=new char[MaxSizeToRead];
	fseeko64(TheFile, Offset, SEEK_SET); //Seek to the beginning write offset of our file
	#ifdef WIN32 //STDIN only needs to be set to binary mode in windows
		_setmode(_fileno(stdin), _O_BINARY);
	#endif
	do
	{
		size_t AmountRead=fread(Buffer, 1, MaxSizeToRead, stdin); //Read the data from STDIN
		fwrite(Buffer, AmountRead, 1, TheFile); //Write the data to the file
	}
	while(!feof(stdin)); //Keep reading and writing until STDIN is complete

	//Cleanup
	delete[] Buffer;
	fclose(TheFile);
	return 1;
}

I needed a command line utility for Bash (for both Windows and Linux) that only outputs bytes between 2 points in a file to STDOUT. head and tail weren’t really cutting it so I figured I’d throw something together. Below is the source code for the result, which I call chunk (Windows Executable).

I compiled the file as c++, but it should be c99 compatible. The file has been tested as compilable for: GCC4 on Slackware, GCC3 on Red Hat, and GCC3 on MingW (Windows [WIN32 should be defined by the compiler]).

Chunk outputs bytes between 2 points in a file to STDOUT. The parameters are:
1) The file
2) The byte offset to start at (hex is supported like 0xA)
3) The number of bytes to output. If not given, the end of the file is assumed.

//Copyright 2009 by Dakusan (http://www.castledragmire.com/Copyright). Licensed under Dakusan License v2.0 (http://www.castledragmire.com/Misc/Software_Licenses/Dakusan_License_v2.0.php).
//See http://www.castledragmire.com/Posts/chunk for more information

#define __LARGE64_FILES
#include <stdio.h>
#include <stdlib.h> //strtoull

#ifdef WIN32 //STDOUT only needs to be set to binary mode in windows
	#include <io.h> //_setmode
	#include <fcntl.h> //_O_BINARY
#endif

typedef unsigned long long UINT64;
const UINT64 MaxSizeToRead=1024*1024*10; //The maximum number of bytes to read at a time to our buffer (Must be < 2^31)

UINT64 GetNumberFromString(const char* S) //Extract both hexidecimal and decimal numbers from a string
{
	bool IsHex=S[0]=='0' && (S[1]|32=='x'); //If string starts as 0x, then is a hex number
	return strtoull(S+(IsHex ? 2 : 0), NULL, IsHex ? 16 : 10); //Hex number starts after 2 characters and uses base 16
}

int main(int argc, char *argv[], char *envp[])
{
	//Determine if proper number of parameters are passed, and if not, output help info
	if(argc!=3 && argc!=4)
		return fprintf(stderr, "Chunk outputs bytes between 2 points in a file to STDOUT. The parameters are:\n1) The file\n2) The byte offset to start at (hex is supported like 0xA)\n3) The number of bytes to output. If not given, the end of the file is assumed.\n") & 0;

	//Open the file and get its length
	FILE *TheFile=fopen64(argv[1], "rb");
	if(TheFile==NULL)
		return fprintf(stderr, "File not found or cannot open file\n") & 0;
	fseeko64(TheFile, 0, SEEK_END); //Get the length by seeking to the end
	UINT64 FileSize=ftello64(TheFile);

	//Determine the requested start offset
	UINT64 Offset=GetNumberFromString(argv[2]), SizeToOutput;
	if(Offset>=FileSize)
	{
		fprintf(stderr, "Offset is larger than file's size\n");
		fclose(TheFile);
		return 0;
	}

	//Determine the size to read
	if(argc==3) //If no final parameter, read to the end of the file
		SizeToOutput=FileSize-Offset;
	else //Determine from the 3rd parameter
	{
		SizeToOutput=GetNumberFromString(argv[3]);
		if(Offset+SizeToOutput>FileSize)
		{
			fprintf(stderr, "Requested size is larger than the file, truncating to end of file\n");
			SizeToOutput=FileSize-Offset;
		}
		else if(!SizeToOutput) //If nothing to output, exit prematurely
		{
			fclose(TheFile);
			return 1;
		}
	}

	//Output requested data 10MB at a time from the file to STDOUT
	char *Buffer=new char[SizeToOutput>MaxSizeToRead ? MaxSizeToRead : SizeToOutput]; //Only allocate as many bytes to our read buffer as is necessary
	fseeko64(TheFile, Offset, SEEK_SET); //Seek to the beginning read offset of our file
	#ifdef WIN32 //STDOUT only needs to be set to binary mode in windows
		_setmode(_fileno(stdout), _O_BINARY);
	#endif
	while(SizeToOutput) //Keep reading and outputting until requested data is complete
	{
		UINT64 SizeToRead=SizeToOutput>MaxSizeToRead ? MaxSizeToRead : SizeToOutput; //Number of bytes to read and write
		fread(Buffer, SizeToRead, 1, TheFile); //Read the data
		fwrite(Buffer, SizeToRead, 1, stdout); //Write the data to STDOUT
		SizeToOutput-=SizeToRead; //Decrease number of bytes we still need to read
	}

	//Cleanup
	delete[] Buffer;
	fclose(TheFile);
	return 1;
}