The House of Software Engineer

Two versions for Lock-Free Stack

Tue, 24 Jun 2008 11:09:06 GMT

The two codes below are implementations of lock-free stack structure.

The purpose is the same, but the first version can be ported easily to other OSes.

Performance benchmark is a lesson to the reader.

The raw C++ implementation. In C++0x we'll change Interlocked APIs to atomic operations:

template

<typename T>
struct LockFreeStack
{
LockFreeStack() : Head_( NULL ){}

void Push( T& value )
{
    PNODE node = new NODE( value );
    PNODE oldHead;

    do
    {
      node->Next = oldHead = Head_;
    }while( oldHead != reinterpret_cast<PNODE>( InterlockedCompareExchangePointer(
                                      reinterpret_cast<volatile PVOID*>(&Head_), node, oldHead )));
}

T Pop()
{
    PNODE node;
    PNODE oldHead;

    do
    {
      oldHead = Head_;
    }while( oldHead != (node = reinterpret_cast<PNODE>( InterlockedCompareExchangePointer(
                                            reinterpret_cast<volatile PVOID*>(&Head_), Head_->Next, oldHead ))));

    T temp = node->Data;
    delete node;
    return temp;
}

private:
typedef struct NODE_TAG
{
    NODE_TAG( T value ) : Next(NULL), Data(value){}
    NODE_TAG* Next;
    T Data;
} NODE, *PNODE;

PNODE Head_;

LockFreeStack( const LockFreeStack& ){}
LockFreeStack& operator=( const LockFreeStack& ){}
};

Using a Windows API facility called Singly Linked Lists:

template<typename T>
struct LockFreeStack
{
LockFreeStack()
{
    Head_ = new_aligned<SLIST_HEADER>();
    if( NULL == Head_ )
      throw "memory allocation failed";

    InitializeSListHead( Head_ );
}

~LockFreeStack()
{
    InterlockedFlushSList( Head_ );
}

void Push( T value )
{
    Item_ = new_aligned<ITEM>();
    if( NULL == Item_ )
      throw "stack is full";

    Item_->Value = value;
    Entry_ = InterlockedPushEntrySList( Head_, &Item_->Entry );
}

T Pop()
{
    PSLIST_ENTRY tempEntry = InterlockedPopEntrySList( Head_ );
    if( NULL == tempEntry )
      throw "stack is empty";

    Item_ = reinterpret_cast<PITEM>( tempEntry );
    T temp = Item_->Value;
    _aligned_free( tempEntry );
    return temp;
}

private:
typedef struct ITEM_TAG
{
    SLIST_ENTRY Entry;
    T Value;
} ITEM, *PITEM;

PSLIST_ENTRY Entry_;
PSLIST_HEADER Head_;
PITEM Item_;

template <class M> M* new_aligned()
{
    return reinterpret_cast<M*>(_aligned_malloc( sizeof(M), MEMORY_ALLOCATION_ALIGNMENT ));
}

LockFreeStack( const LockFreeStack& ){}
LockFreeStack& operator=( const LockFreeStack& ){}
};

Samples from my Concurrency Talk at Seminário TempoReal C++ Portabilidade e Performance

Fri, 13 Jun 2008 13:11:13 GMT

Samples from my Concurrency Talk at Seminário TempoReal C++ Portabilidade e Performance:

Concurrency C++ Samples
Concurrency C++ Slides (in brazilian portuguese)

The following code is a C++ port to one of my previous samples: MapReduce with Parallelspace

#include

<iostream>
#include <fstream>
#include <vector>
#include <map>
#include <string>
#include <boost/thread/thread.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/filesystem.hpp>
#include <boost/function.hpp>
#include <boost/algorithm/string.hpp>

using namespace std;
using namespace boost;
using namespace boost::filesystem;

void get_files( const path& directory, vector<path>& files )
{
directory_iterator end_iter;
for( directory_iterator iter( directory ); iter != end_iter; ++iter )
{
if( is_directory( iter->status() ) ) continue;
files.push_back( iter->path() );
}
}

int get_ProcessorCount(){ return 2; }

class TaskExecutor
{
typedef function2<void, map<string, int>&, const map<string, int>&> ConsolidationFunctionType;
typedef function1<map<string, int>, const path&> CountWordsFunctionType;

public:
TaskExecutor( vector<path>::const_iterator begin, vector<path>::const_iterator end,
                      const CountWordsFunctionType countWords,
                      const ConsolidationFunctionType consolidation,
                      map<string, int>& result ) : Begin_( begin ),
                      End_( end ),
                      CountWordsFunction_( countWords ),
                      ConsolidationFunction_( consolidation ),
                      Result_( result ){}

void operator()()
{
for( vector<path>::const_iterator iter = Begin_; iter != End_; ++iter )
ConsolidationFunction_( Result_, CountWordsFunction_( *iter ) );
}

private:
vector<path>::const_iterator Begin_, End_;
const ConsolidationFunctionType ConsolidationFunction_;
const CountWordsFunctionType CountWordsFunction_;
map<string, int>& Result_;
};

void key_count( map<string, int>& dictionary, const string& word, int value )
{
dictionary[ word ] = dictionary.end() != dictionary.find( word ) ? dictionary[ word ] + value : value;
}

map<string, int> map_function( const path& filename )
{
map<string, int> wordCount;

ifstream file;
const int MAXLEN = 1024;
char line[ MAXLEN ];

file.open( filename.directory_string().c_str() );

while( file.getline( line, MAXLEN ) )
{
    vector<string> split_v;
    split( split_v, string( line ), is_any_of( " " ) );
    for( vector<string>::const_iterator iter = split_v.begin(); iter != split_v.end(); ++iter )
      key_count( wordCount, *iter, 1 );
}

file.close();

return wordCount;
}

void reduce_function( map<string, int>& destination, const map<string, int>& source )
{
for( map<string, int>::const_iterator iter = source.begin(); iter != source.end(); ++iter )
key_count( destination, iter->first, iter->second );
}

void wordcount_mapreduced( const vector<path>& files, map<string, int>& wordCount )
{
//data partition
int numberOfPartitions = get_ProcessorCount() * 2; //2 threads per core
int numberOfFiles = static_cast<int>( files.size() );
if( numberOfFiles < numberOfPartitions ) numberOfPartitions = numberOfFiles;
int delta = numberOfFiles / numberOfPartitions;
vector< map<string, int> > result( numberOfPartitions );
thread_group tg;

//for parallel
for( int step = 0; step < numberOfPartitions; ++step )
{
    vector<path>::const_iterator begin = files.begin() + delta * step,
    end = numberOfPartitions - 1 == step ? files.end() : files.begin() + delta * step + delta;
    //fork
    tg.create_thread( TaskExecutor( begin, end, map_function, reduce_function, result[step] ) );
}

//join
tg.join_all();
for( vector< map<string, int> >::const_iterator iter = result.begin(); iter != result.end(); ++iter )
reduce_function( wordCount, *iter );
}

int main()
{
vector<path> files;
get_files( "c:\\txttest\\", files );
map<string, int> wordCount;

wordcount_mapreduced( files, wordCount );

for( map<string, int>::const_iterator iter = wordCount.begin(); iter != wordCount.end(); ++iter )
cout << iter->first << " = " << iter->second << endl;
}

Get the size of directories from all drives with F#

Mon, 25 Feb 2008 14:48:24 GMT

This small F# utility uses recursion to inspect all directories from all drives installed (including network drives mapped).

The sizes are printed in the console screen (or could be redirected).

#light

open System.IO
open System

let ConvertTo (n:int64) (d:float) = n / Convert.ToInt64( d )
//let ConvertToMegabytes (n:int64) = ConvertTo n (1024.0 ** 2.0)
let ConvertToKilobytes (n:int64) = ConvertTo n 1024.0

let rec getSizeOfDir (path:string) =
let getSizeOfFiles (dir:string) =
        try
            DirectoryInfo(dir).GetFiles() |> Seq.sumByInt64( fun file -> file.Length )
        with
            | :? UnauthorizedAccessException -> 0L;

    try
        let size = (Directory.GetDirectories( path ) |> Seq.sumByInt64( fun dir -> getSizeOfDir( dir ) )) + getSizeOfFiles( path )
        printfn "Path = %s\r\nSize = %d KB (%d bytes)\r\n" path (ConvertToKilobytes size) size
        size
    with
        | :? UnauthorizedAccessException -> 0L
        | :? IOException -> 0L;

let main =
    let sw = new Diagnostics.Stopwatch()
    let tm = DateTime.Now
    sw.Start()
    DriveInfo.GetDrives() |> Seq.iter( fun drv -> getSizeOfDir drv.Name |> ignore )
    sw.Stop()
    printfn "\r\n\r\nElapsed time = %s" (sw.Elapsed.ToString())

This utility could run better if reworked to support threads. Maybe next time...

XNA, Visual C++ and Guitar Hero

Thu, 07 Feb 2008 11:43:13 GMT

Is it possible to write XNA programs with Visual C++? Yes, it is! Someone show that is possible in F#, too.

The XNA Game Studio 2.0 can be installed inside Visual Studio 2005 Professional (the previous version is hosted only in VC# Express).

The sample below is a Guitar Hero Panel. You plug-in a guitar on PC, and press Fret buttons, Strum bar and Whammy bar to see the feedback. (This sample doesn´t have sounds, just visual feedback)

How can you do to write your own XNA programs with Visual C++?

1. Create a XNA Windows Game (or XNA Xbox 360 Game) project with C#. (C# will be the host of the C++/CLI dll. C# is necessary to compile Game Assets like images, fonts, ...)

2. Add a new project Visual C++ CLR Class Library into the current solution. The XNA program will be coded entirely in C++/CLI, here. Is very important (if you want to port to Xbox 360) to compile this project with /clr:safe switch. This ensure that the generated code is MSIL-only. Add reference to the assemblies: Microsoft.Xna.Framework.dll and Microsoft.Xna.Framework.Game.dll. They are at Microsoft XNA install dir something like: <XNA install dir>\XNA Game Studio\v2.0\References\Windows\x86\

3. Add a reference to the VC++ CLR DLL in the C# project. And call the game class written in C++/CLI, at Main method in C# code. (You even have the ability to debug step-by-step)

The following code is the C++/CLI game program. If you look with attention you see that we have templates and stack semantics:

using namespace Microsoft::Xna::Framework;
using namespace Microsoft::Xna::Framework::Graphics;
using namespace Microsoft::Xna::Framework::Input;

namespace GuitarHeroControlPanel
{

ref struct Piece
{
    Piece( Texture2D^ image, Vector2 position, SpriteBatch^ spriteBatch ) :
    Display_( false ), Image_( image ), Position_( position ), SpriteBatch_( spriteBatch ){}

    void Update( bool displayStatus ){ Display_ = displayStatus; }
    void Draw(){ if( Display_ ) SpriteBatch_->Draw( Image_, Position_, Color::White ); }

private:
    bool Display_;
    Vector2 Position_;
    Texture2D^ Image_;
    SpriteBatch^ SpriteBatch_;
};

public ref class GuitarHeroControlPanel : Game
{
public:
GuitarHeroControlPanel()
    {
        Graphics_ = gcnew GraphicsDeviceManager(this);
        Graphics_->PreferredBackBufferWidth = 1280;
        Graphics_->PreferredBackBufferHeight = 416;
        Window->Title = "Guitar Hero Panel by Fabio Galuppo. Implemented with XNA and Visual C++ 2005. Visit: http://www.gamecultura.com.br/. Blog: http://fabiogaluppo.spaces.live.com/";
        Content->RootDirectory = "Content";
}

private:
    template<class T> T^ CntLoad( System::String^ assetName ){ return Content->Load<T^>( assetName ); }
    template<PlayerIndex P> GamePadButtons GetButtons(){ return GamePad::GetState(P).Buttons; }
    template<PlayerIndex P> GamePadDPad GetDPad(){ return GamePad::GetState(P).DPad; }
    template<PlayerIndex P> GamePadThumbSticks GetThumbSticks(){ return GamePad::GetState(P).ThumbSticks; }
    bool IsPressed( ButtonState bs ){ return bs == ButtonState::Pressed; }

protected:
    virtual void LoadContent() override
    {
        SpriteBatch_ = gcnew SpriteBatch(GraphicsDevice);
        Font_ = CntLoad<SpriteFont>( "Tahoma" );
        GuitarTexture_ = CntLoad<Texture2D>( "guitarhero" );
        GCLogo_ = CntLoad<Texture2D>( "gclogo" );
        VCLogo_ = CntLoad<Texture2D>( "vclogo" );

        GreenFret_ = gcnew Piece( CntLoad<Texture2D>( "green" ), Vector2(965.0, 0.0), SpriteBatch_ );
        RedFret_ = gcnew Piece( CntLoad<Texture2D>( "red" ), Vector2(930.0, 0.0), SpriteBatch_ );
        YellowFret_ = gcnew Piece( CntLoad<Texture2D>( "yellow" ), Vector2(895.0, 0.0), SpriteBatch_ );
        BlueFret_ = gcnew Piece( CntLoad<Texture2D>( "blue" ), Vector2(858.0, 0.0), SpriteBatch_ );
        OrangeFret_ = gcnew Piece( CntLoad<Texture2D>( "orange" ), Vector2(821.0, 0.0), SpriteBatch_ );
        Strum_ = gcnew Piece( CntLoad<Texture2D>( "strum" ), Vector2(269.0, 0.0), SpriteBatch_ );
        Whammy_ = gcnew Piece( CntLoad<Texture2D>( "whammy" ), Vector2(0.0, 259.0), SpriteBatch_ );

        GCLogoVec_ = Vector2( 895.0 , 0.0 );
        VCLogoVecPos_ = Vector2( 250.0, 130.0 );
        VCLogoVecOri_ = Vector2( VCLogo_->Width / 2.0f, VCLogo_->Height / 2.0f );
        FontVec_ = Vector2( 880.0, 380.0 );
    }

    virtual void Update(GameTime^ gameTime) override
    {
        if ( IsPressed( GetButtons<PlayerIndex::One>().Back ) ) Game::Exit();

        //Fret Buttons
        GreenFret_->Update( IsPressed( GetButtons<PlayerIndex::One>().A ) );
        RedFret_->Update( IsPressed( GetButtons<PlayerIndex::One>().B ) );
        YellowFret_->Update( IsPressed( GetButtons<PlayerIndex::One>().Y ) );
        BlueFret_->Update( IsPressed( GetButtons<PlayerIndex::One>().X ) );
        OrangeFret_->Update( IsPressed( GetButtons<PlayerIndex::One>().LeftShoulder ) );

        //Strum Bar
        Strum_->Update( IsPressed( GetDPad<PlayerIndex::One>().Up ) || IsPressed( GetDPad<PlayerIndex::One>().Down ) );

        //Whammy Bar
        Whammy_->Update( GetThumbSticks<PlayerIndex::One>().Right.X >= 0.0 );

        Game::Update(gameTime);
    }

    virtual void Draw(GameTime^ gameTime) override
    {
        Graphics_->GraphicsDevice->Clear(Color::CornflowerBlue);
        Game::Draw(gameTime);

        SpriteBatch_->Begin();

            SpriteBatch_->Draw( GuitarTexture_, Vector2::Zero, Color::White );

            GreenFret_->Draw();
            RedFret_->Draw();
            YellowFret_->Draw();
            BlueFret_->Draw();
            OrangeFret_->Draw();
            Strum_->Draw();

            Whammy_->Draw();

            SpriteBatch_->Draw( GCLogo_, GCLogoVec_, Color::White );

            SpriteBatch_->Draw( VCLogo_, VCLogoVecPos_, NullRect_, Color::White, -45.0, VCLogoVecOri_, 1.0, SpriteEffects::None, 0.0 );

            SpriteBatch_->DrawString( Font_, "http://fabiogaluppo.spaces.live.com/", FontVec_, Color::Black );

        SpriteBatch_->End();
    }

private:
    GraphicsDeviceManager^ Graphics_;
    SpriteBatch^ SpriteBatch_;
    Texture2D ^GuitarTexture_, ^GCLogo_, ^VCLogo_;
    Vector2 GCLogoVec_, VCLogoVecPos_, VCLogoVecOri_, FontVec_;
    System::Nullable<Rectangle> NullRect_;
    SpriteFont^ Font_;
    Piece ^GreenFret_, ^RedFret_, ^YellowFret_, ^BlueFret_, ^OrangeFret_, ^Strum_, ^Whammy_;
};

};

The C# host program:

using System;

namespace GuitarHeroControlPanel
{

static class Program
{
static void Main()
    {
        using (GuitarHeroControlPanel panel = new GuitarHeroControlPanel()) panel.Run();
    }
}

}

Download the solution to GHP

Do you like games? Stay tuned at GameCultura

TechEd Brasil 2007 - TxF Demo #5

Thu, 03 Jan 2008 13:12:02 GMT

How to consume the TxF wrapper with C#:

using System;
using System.Transactions;
using System.IO;

class Program
{
    static void Main(string[] args)
    {
        using (TransactionScope scope = new TransactionScope())
        {
            using (StreamWriter sw = new StreamWriter(NTFS.Transactional.File.OpenTransacted("c:\\ManagedNTFSTx.txt", FileMode.OpenOrCreate, FileAccess.Write, FileShare.None)))
                for (int i = 0; i < 1000; ++i) sw.WriteLine("Hello World");

            scope.Complete(); //omit this line to rollback transaction
        }

        using (TransactionScope scope = new TransactionScope())
        {
            NTFS.Transactional.File.CopyTransacted("c:\\ManagedNTFSTx.txt", "c:\\ManagedNTFSTxClone.txt", true);

            scope.Complete(); //omit this line to rollback transaction
        }
    }
}

TechEd Brasil 2007 - TxF Demo #4

Thu, 03 Jan 2008 12:57:33 GMT

This managed wrapper is inspired/adapted/improved from this article.

It supports some NTFS transactional file operations like copy, delete, create and open.

using

System;
using System.IO;
using System.Security;
using System.Runtime.InteropServices;
using System.Runtime.ConstrainedExecution;
using System.Runtime.CompilerServices;
using Microsoft.Win32.SafeHandles;
using System.Transactions;

namespace NTFS.Transactional
{
    [ComImport, Guid("79427A2B-F895-40e0-BE79-B57DC82ED231"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
    interface IKernelTransaction
    {
        int GetHandle(out IntPtr pHandle);
    }

    [SuppressUnmanagedCodeSecurity]
    static class WindowsAPI
    {
        [DllImport("kernel32.dll", SetLastError = true)]
        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)]
        public static extern bool CloseHandle(IntPtr hObject);

        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
        public static extern SafeFileHandle CreateFileTransacted
        (
            string lpFileName,
            Constants.FileAccess dwDesiredAccess,
            Constants.FileShare dwShareMode,
            IntPtr lpSecurityAttributes,
            Constants.FileMode dwCreationDisposition,
            int dwFlagsAndAttributes,
            IntPtr hTemplateFile,
            IntPtr hTransaction,
            IntPtr pusMiniVersion,
            IntPtr pExtendedParameter
        );

        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
        public static extern bool CopyFileTransacted
        (
            string lpExistingFileName,
            string lpNewFileName,
            IntPtr lpProgressRoutine,
            IntPtr lpData,
            ref bool pbCancel,
            Constants.CopyFile dwCopyFlags,
            IntPtr hTransaction
        );

        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
        public static extern bool DeleteFileTransacted
        (
            string file, IntPtr transaction
        );

        public static bool SUCCEDDED(int hr) { return hr == Constants.ERROR_SUCCESS; }

        public static void ThrowsLastError() { throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); }

        public static class Constants
        {
            public const int ERROR_SUCCESS = 0x0;

            [Flags]
            public enum FileAccess
            {
                GENERIC_READ = unchecked((int)0x80000000),
                GENERIC_WRITE = 0x40000000
            }

            [Flags]
            public enum FileShare
            {
                FILE_SHARE_NONE = 0x0,
                FILE_SHARE_READ = 0x1,
                FILE_SHARE_WRITE = 0x2,
                FILE_SHARE_DELETE = 0x4
            }

            public enum FileMode
            {
                CREATE_NEW = 0x1,
                CREATE_ALWAYS = 0x2,
                OPEN_EXISTING = 0x3,
                OPEN_ALWAYS = 0x4,
                TRUNCATE_EXISTING = 0x5
            }

            [Flags]
            public enum CopyFile
            {
                COPY_FILE_FAIL_IF_EXISTS = 0x1,
                COPY_FILE_RESTARTABLE = 0x2,
                COPY_FILE_OPEN_SOURCE_FOR_WRITE = 0x4,
                COPY_FILE_ALLOW_DECRYPTED_DESTINATION = 0x8,
                COPY_FILE_COPY_SYMLINK = 0x800
            }
        }
    }

    public static class File
    {
        public static void DeleteTransacted(string path)
        {
            using (KernelTransactionHandle transactionHandle = KernelTransactionHandle.Get())
                if (!WindowsAPI.DeleteFileTransacted(path, transactionHandle.DangerousGetHandle()))
                    WindowsAPI.ThrowsLastError();
        }

        public static void CopyTransacted(string from, string to, bool overwrite)
        {
            using (KernelTransactionHandle transactionHandle = KernelTransactionHandle.Get())
            {
                bool cancel = false;
                if (!WindowsAPI.CopyFileTransacted
                        (
                            from,
                            to,
                            IntPtr.Zero,
                            IntPtr.Zero,
                            ref cancel,
                            overwrite ? 0 : WindowsAPI.Constants.CopyFile.COPY_FILE_FAIL_IF_EXISTS,
                            transactionHandle.DangerousGetHandle()
                        )
                )
                    WindowsAPI.ThrowsLastError();
            }
        }

        public static FileStream OpenTransacted(string path, FileMode mode, FileAccess access, FileShare share)
        {
            using (KernelTransactionHandle transactionHandle = KernelTransactionHandle.Get())
            {
                SafeFileHandle fileTransactedHandle = WindowsAPI.CreateFileTransacted
                                                                        (
                                                                          path,
                                                                            ToNativeFileAccess(access),
                                                                            ToNativeFileShare(share),
                                                                            IntPtr.Zero,
                                                                            ToNativeFileMode(mode),
                                                                            0,
                                                                            IntPtr.Zero,
                                                                            transactionHandle.DangerousGetHandle(),
                                                                            IntPtr.Zero,
                                                                            IntPtr.Zero
                                                                        );

                if (fileTransactedHandle.IsInvalid)

                    WindowsAPI.ThrowsLastError();
                return new FileStream(fileTransactedHandle, access);
            }
        }

        static WindowsAPI.Constants.FileAccess ToNativeFileAccess(FileAccess access)
        {
            if (FileAccess.Read == access)
                return WindowsAPI.Constants.FileAccess.GENERIC_READ;
            else if (FileAccess.Write == access)
                return WindowsAPI.Constants.FileAccess.GENERIC_WRITE;
            return WindowsAPI.Constants.FileAccess.GENERIC_READ | WindowsAPI.Constants.FileAccess.GENERIC_WRITE;
        }

        static WindowsAPI.Constants.FileMode ToNativeFileMode(FileMode mode)
        {
            return (WindowsAPI.Constants.FileMode)(int)(FileMode.Append == mode ? FileMode.OpenOrCreate : mode);
        }

        static WindowsAPI.Constants.FileShare ToNativeFileShare(FileShare share)
        {
            return (WindowsAPI.Constants.FileShare)(int)share;
        }
    }

    public class KernelTransactionHandle : SafeHandleZeroOrMinusOneIsInvalid
    {
        internal KernelTransactionHandle(IntPtr existingHandle) : this(existingHandle, true) { }
        internal KernelTransactionHandle(IntPtr existingHandle, bool ownsHandle) : base(true)
        {
            this.SetHandle(existingHandle);
        }

        protected override bool ReleaseHandle() { return WindowsAPI.CloseHandle(handle); }

        public static KernelTransactionHandle Get() { return Get(Transaction.Current); }
        public static KernelTransactionHandle Get(Transaction transaction)
        {
            if (null == transaction)
                throw new NullReferenceException();

            IKernelTransaction kernelTransaction = TransactionInterop.GetDtcTransaction(transaction) as IKernelTransaction;
            KernelTransactionHandle transactionHandler = null;

            RuntimeHelpers.PrepareConstrainedRegions();

            try { } //CER Scope

            finally
            {
                IntPtr txHandle;
                int hr = kernelTransaction.GetHandle(out txHandle);

                if (!WindowsAPI.SUCCEDDED(hr))

throw new System.ComponentModel.Win32Exception(hr);

                transactionHandler = new KernelTransactionHandle(txHandle);
            } //CER Scope

            return transactionHandler;
        }
    }
}

Web Page request asynchronously with Parallelspace

Fri, 28 Dec 2007 20:09:25 GMT

Today, I released the Parallelspace version 0.2. The following sample is a classic implementation of many threads requesting web pages.

This new version has a lot of new stuff like Async I/O operations via Begin/End methods. In this sample, AsyncUtil.WebRequestGetResponse is responsable to call the Async I/O operations.

using System;
using System.Threading;
using System.Net;

using

Parallelspace;
using Parallelspace.Utilities;

class WebRequestResponseAsyncSample
{
    private void UseProxy(WebRequest req, string proxyAddress, string user, string pwd)
    {
        WebProxy proxy = new WebProxy(proxyAddress);
        proxy.Credentials = new NetworkCredential(user, pwd);
        req.Proxy = proxy;
    }

    private void Display( string url, string content )
    {
        String line = new String('-', url.Length + 5);
        System.Text.StringBuilder sb = new System.Text.StringBuilder();
        sb.AppendFormat("{0}\r\n", line);
        sb.AppendFormat("url: {0}\r\n", url);
        sb.AppendFormat("thread id: {0}\r\n", Thread.CurrentThread.ManagedThreadId);
        sb.AppendFormat("{0}\r\n", line);
        sb.Append(content);
        sb.AppendFormat("\r\n{0}\r\n", line);
        Console.WriteLine(sb.ToString());
    }

    public Future UrlGetAsync(Block block, string url)
    {
        return block.Run<DataWithWaitHandle<WebResponse, ManualResetEvent>>(
            delegate
            {
                try
                {
                    WebRequest req = WebRequest.Create(url);
                    //UseProxy(req, "proxy_address", "user", "password" );
                    return new DataWithWaitHandle<WebResponse, ManualResetEvent>
                        (
                            AsyncUtil.WebRequestGetResponse(req),
                            new ManualResetEvent(false)
                        );
                }
                catch (Exception e)
                {
                    Display(url, e.Message);
                    return new DataWithWaitHandle<WebResponse, ManualResetEvent>(null,
                                    new ManualResetEvent(true));
                }
            },

            delegate(Future<DataWithWaitHandle<WebResponse, ManualResetEvent>> f)
            {
                DataWithWaitHandle<WebResponse, ManualResetEvent> value = f.Value;

                if (null != value.Data)
                    using (System.IO.StreamReader sr = new System.IO.StreamReader(value.Data.GetResponseStream()))
                        Display(url, sr.ReadToEnd().Substring(0, 1024));

                value.WaitHandle.Set();
            },

            url
        );
    }

    public void Run( params string[] urls )
    {
        if (null != urls && urls.Length > 0)
        {
            Future[] f = new Future[urls.Length];
            for (int i = 0; i < urls.Length; ++i)
                f[i] = UrlGetAsync(BlockRunner.Get(), urls[i]);
            WaitHandle.WaitAll(ConvertUtil.ToWaitHandleArray<WebResponse, ManualResetEvent>(f));
        }
    }
}

class Program
{
    static void Main(string[] args)
    {
        //new WebRequestResponseAsyncSample().Run(args);

        new WebRequestResponseAsyncSample().Run(
                    "http://www.thisisnotsupposedtoexist.org/",
                    "http://www.live.com/",
                    "http://blogs.msdn.com/MSRoboticsStudio/",
                    "http://fabiogaluppo.spaces.live.com/",
                    "This isn't an URL"
            );
    }
}

A cool implementation in F# can be found in Don Syme's blog (the main designer of F#). Another implementation, using CCR, was presented at MSDN Magazine by Jeffrey Richter.

MapReduce with Parallelspace

Mon, 24 Dec 2007 20:18:44 GMT

I was reading about MapReduce here and there, and I remember that I had posted something about MapReduce at my old house.

Techniques applied in MapReduce gives me a great opportunity to show concurrency with Parallelspace (of course, it can be done with any other library).

In my point of view, MapReduce is a pure application of Fork/Join parallel pattern. So, you must: 1. create a data partition - breaking this data in pieces; 2. spawn tasks (fork) and define how to use these chunks producing a partial result (map); 3. wait for tasks to complete (join); and 4. consolidate the results in the end (reduce). The following C# 2.0 code, demonstrate a word count program applying MapReduce with Parallelspace.

using System;
using System.IO;
using System.Collections.Generic;

using Parallelspace;
using Parallelspace.Schedulers;

sealed class WordCount
{
    public delegate void Action2<T0, T1>(T0 p0, T1 p1);

    public Dictionary<string, long> MappedReduced(string[] files, ReturnAction<string, Dictionary<string, long>> countWords, Action2<Dictionary<string, long>, Dictionary<string, long>> consolidation)
    {
        int numberOfPartitions = Environment.ProcessorCount * 2; //2 threads per core

        if (files.Length < numberOfPartitions) numberOfPartitions = files.Length;
        int delta = files.Length / numberOfPartitions;

        Block asyncBlock = new Block(new ThreadScheduler(true));
        Future[] futures = new Future[numberOfPartitions];

        //"Parallel.For"
        for (int step = 0; step < numberOfPartitions; ++step)
        {
            int begin = delta * step,
                 end  = numberOfPartitions - 1 == step ? files.Length : begin + delta;

            //Fork
            futures[step] = asyncBlock.Run<Dictionary<string, long>>(
                delegate
                {
                    Dictionary<string, long> temp = new Dictionary<string, long>();
                    for (int i = begin; i < end; ++i) consolidation(temp, countWords(files[i]));
                    return temp;
                });
        }

        //Join
        Dictionary<string, long> wordCount = new Dictionary<string,long>();
        foreach (Future<Dictionary<string, long>> f in futures)
        {
            Dictionary<string, long> v = f.Value;
            consolidation(wordCount, f.Value);
        }
        return wordCount;
    }
}

class Program
{
    static string[] Split(string line)
    {
        return line.Split(new char[]{' '}, StringSplitOptions.RemoveEmptyEntries);
    }

    static void KeyCount(Dictionary<string, long> dictionary, string key, long value)
    {
        dictionary[key] = dictionary.ContainsKey(key) ? dictionary[key] + value : value;
    }

    static void Dump(Dictionary<string, long> result)
    {
        foreach (KeyValuePair<string, long> entry in result)
            Console.Write("[{0} = {1}] ", entry.Key, entry.Value);
    }

    static Dictionary<string, long> WordCountMappedReduced(string directory, string searchPattern)
    {
        return new WordCount().MappedReduced
            (
                Directory.GetFiles(directory, searchPattern),

                /* map method */
                delegate(string file)
                {
                    Dictionary<string, long> wordCount = new Dictionary<string, long>();
                    using (StreamReader sr = new StreamReader(File.OpenRead(file)))
                    {
                        while (sr.Peek() >= 0)
                            foreach (string word in Split(sr.ReadLine()))
                                KeyCount(wordCount, word, 1);
                            return wordCount;
                    }
                },

                /* reduce method */
                delegate(Dictionary<string, long> destination, Dictionary<string, long> source)
                {
                    foreach (KeyValuePair<string, long> entry in source)
                        KeyCount(destination, entry.Key, entry.Value);
                }
            );
    }

    static void Main(string[] args)
    {
        //args: 0 = directory; 1 = search pattern;
        Dictionary<string, long> result = WordCountMappedReduced(args[0], args[1]);
        Dump(result);
    }
}

Did you perceived this is a lock-free implementation?

The Google´s MapReduce was VERY influenced by map and reduce features from LISP. Obviously, the Google framework is more than a programming style. But, the MapReduce (or Fork/Join) can be implemented with any functional programming languages (or even non-functional programming languages). Virtually, through Delegate, any .NET language support a functional-style of programming. For each new release, C# is incorporating more and more functional programming elements (lambda expressions, type inference, closures, ...). And C++, in the future (0x), will be supporting lambda as first-class language feature! (I hope)