Imports System.Collections.Generic
Imports Microsoft.Office.Tools
Imports Office = Microsoft.Office.Core
Imports Outlook = Microsoft.Office.Interop.Outlook

Public Class InspectorWrapper
    Private inspector As Outlook.Inspector
    Private WithEvents inspectorEvents As Outlook.InspectorEvents_Event
    Private WithEvents taskPane As CustomTaskPane

    Public Sub New(ByVal Inspector As Outlook.Inspector)
        Me.inspector = Inspector
        inspectorEvents = TryCast(Me.inspector, Outlook.InspectorEvents_Event)
        taskPane = Globals.ThisAddIn.CustomTaskPanes.Add(New TaskPaneControl(), _
            "My task pane", Inspector)
    End Sub

    Private Sub TaskPane_VisibleChanged(ByVal sender As Object, ByVal e As EventArgs) _
        Handles taskPane.VisibleChanged
        Globals.Ribbons(inspector).ManageTaskPaneRibbon.ToggleButton1.Checked = taskPane.Visible
    End Sub

    Sub InspectorWrapper_Close() Handles inspectorEvents.Close
        If Not (taskPane Is Nothing) Then
            Globals.ThisAddIn.CustomTaskPanes.Remove(taskPane)
        End If

        taskPane = Nothing
        Globals.ThisAddIn.InspectorWrappers.Remove(inspector)
        RemoveHandler inspectorEvents.Close, AddressOf InspectorWrapper_Close
        inspector = Nothing
    End Sub

    Public ReadOnly Property CustomTaskPane() As CustomTaskPane
        Get
            Return taskPane
        End Get
    End Property
End Class

Public Class ThisAddIn

    Private inspectorWrappersValue As New Dictionary(Of Outlook.Inspector, InspectorWrapper)
    Private WithEvents inspectors As Outlook.Inspectors

    Private Sub ThisAddIn_Startup(ByVal sender As Object, ByVal e As System.EventArgs) _
        Handles Me.Startup

        inspectors = Me.Application.Inspectors
        Dim inspector As Outlook.Inspector
        For Each inspector In inspectors
            Inspectors_NewInspector(inspector)
        Next inspector
    End Sub

    Private Sub ThisAddIn_Shutdown(ByVal sender As Object, ByVal e As System.EventArgs) _
        Handles Me.Shutdown

        RemoveHandler inspectors.NewInspector, AddressOf Inspectors_NewInspector
        inspectors = Nothing
        inspectorWrappersValue = Nothing
    End Sub

    Sub Inspectors_NewInspector(ByVal Inspector As Outlook.Inspector) _
        Handles inspectors.NewInspector

        If TypeOf Inspector.CurrentItem Is Outlook.MailItem Then
            inspectorWrappersValue.Add(Inspector, New InspectorWrapper(Inspector))
        End If
    End Sub

    Public ReadOnly Property InspectorWrappers() As Dictionary(Of Outlook.Inspector, InspectorWrapper)
        Get
            Return inspectorWrappersValue
        End Get
    End Property
End Class

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Outlook = Microsoft.Office.Interop.Outlook;
using Office = Microsoft.Office.Core;
using Microsoft.Office.Tools;

namespace OutlookMailItemTaskPane
{
    public class InspectorWrapper
    {
        private Outlook.Inspector inspector;
        private CustomTaskPane taskPane;

        public InspectorWrapper(Outlook.Inspector Inspector)
        {
            inspector = Inspector;
            ((Outlook.InspectorEvents_Event)inspector).Close +=
                new Outlook.InspectorEvents_CloseEventHandler(InspectorWrapper_Close);

            taskPane = Globals.ThisAddIn.CustomTaskPanes.Add(
                new TaskPaneControl(), "My task pane", inspector);
            taskPane.VisibleChanged += new EventHandler(TaskPane_VisibleChanged);
        }

        void TaskPane_VisibleChanged(object sender, EventArgs e)
        {
            Globals.Ribbons[inspector].ManageTaskPaneRibbon.toggleButton1.Checked = 
                taskPane.Visible;
        }

        void InspectorWrapper_Close()
        {
            if (taskPane != null)
            {
                Globals.ThisAddIn.CustomTaskPanes.Remove(taskPane);
            }

            taskPane = null;
            Globals.ThisAddIn.InspectorWrappers.Remove(inspector);
            ((Outlook.InspectorEvents_Event)inspector).Close -=
                new Outlook.InspectorEvents_CloseEventHandler(InspectorWrapper_Close);
            inspector = null;
        }

        public CustomTaskPane CustomTaskPane
        {
            get
            {
                return taskPane;
            }
        }
    }

    public partial class ThisAddIn
    {
        private Dictionary<Outlook.Inspector, InspectorWrapper> inspectorWrappersValue =
            new Dictionary<Outlook.Inspector, InspectorWrapper>();
        private Outlook.Inspectors inspectors;

        private void ThisAddIn_Startup(object sender, System.EventArgs e)
        {
            inspectors = this.Application.Inspectors;
            inspectors.NewInspector +=
                new Outlook.InspectorsEvents_NewInspectorEventHandler(
                Inspectors_NewInspector);

            foreach (Outlook.Inspector inspector in inspectors)
            {
                Inspectors_NewInspector(inspector);
            }
        }

        private void ThisAddIn_Shutdown(object sender, System.EventArgs e)
        {
            inspectors.NewInspector -=
                new Outlook.InspectorsEvents_NewInspectorEventHandler(
                Inspectors_NewInspector);
            inspectors = null;
            inspectorWrappersValue = null;
        }

        void Inspectors_NewInspector(Outlook.Inspector Inspector)
        {
            if (Inspector.CurrentItem is Outlook.MailItem)
            {
                inspectorWrappersValue.Add(Inspector, new InspectorWrapper(Inspector));
            }
        }

        public Dictionary<Outlook.Inspector, InspectorWrapper> InspectorWrappers
        {
            get
            {
                return inspectorWrappersValue;
            }
        }

        #region VSTO generated code

        /// <summary>
        /// Required method for Designer support - do not modify
        /// the contents of this method with the code editor.
        /// </summary>
        private void InternalStartup()
        {
            this.Startup += new System.EventHandler(ThisAddIn_Startup);
            this.Shutdown += new System.EventHandler(ThisAddIn_Shutdown);
        }

        #endregion
    }
}

Imports Microsoft.Office.Tools.Ribbon
Imports Outlook = Microsoft.Office.Interop.Outlook
Imports Microsoft.Office.Tools

Public Class ManageTaskPaneRibbon

    Private Sub ManageTaskPaneRibbon_Load(ByVal sender As System.Object, _
        ByVal e As Microsoft.Office.Tools.Ribbon.RibbonUIEventArgs) Handles MyBase.Load

    End Sub

    Private Sub ToggleButton1_Click(ByVal sender As System.Object, _
        ByVal e As Microsoft.Office.Tools.Ribbon.RibbonControlEventArgs) _
        Handles ToggleButton1.Click

        Dim inspector As Outlook.Inspector = e.Control.Context
        Dim inspectorWrapper As InspectorWrapper = Globals.ThisAddIn.InspectorWrappers(inspector)
        Dim taskPane As CustomTaskPane = inspectorWrapper.CustomTaskPane
        If Not (taskPane Is Nothing) Then
            taskPane.Visible = TryCast(sender, RibbonToggleButton).Checked
        End If
    End Sub
End Class

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.Office.Tools.Ribbon;
using Outlook = Microsoft.Office.Interop.Outlook;
using Microsoft.Office.Tools;

namespace OutlookMailItemTaskPane
{
    public partial class ManageTaskPaneRibbon : OfficeRibbon
    {
        public ManageTaskPaneRibbon()
        {
            InitializeComponent();
        }

        private void ManageTaskPaneRibbon_Load(object sender, RibbonUIEventArgs e)
        {

        }

        private void toggleButton1_Click(object sender, RibbonControlEventArgs e)
        {
            Outlook.Inspector inspector = (Outlook.Inspector)e.Control.Context;
            InspectorWrapper inspectorWrapper = Globals.ThisAddIn.InspectorWrappers[inspector];
            CustomTaskPane taskPane = inspectorWrapper.CustomTaskPane;
            if (taskPane != null)
            {
                taskPane.Visible = ((RibbonToggleButton)sender).Checked;
            }
        }
    }
}